├── test ├── adapters │ ├── mysql2.rb │ ├── jdbcmysql.rb │ ├── postgis.rb │ ├── sqlite3.rb │ ├── postgresql.rb │ ├── spatialite.rb │ ├── jdbcpostgresql.rb │ ├── mysql2spatial.rb │ └── seamless_database_pool.rb ├── models │ ├── question.rb │ ├── rule.rb │ ├── group.rb │ ├── promotion.rb │ ├── discount.rb │ ├── chapter.rb │ ├── end_note.rb │ ├── widget.rb │ ├── book.rb │ └── topic.rb ├── postgis │ └── import_test.rb ├── jdbcpostgresql │ └── import_test.rb ├── mysql2 │ └── import_test.rb ├── jdbcmysql │ └── import_test.rb ├── mysqlspatial2 │ └── import_test.rb ├── schema │ ├── version.rb │ ├── mysql_schema.rb │ └── generic_schema.rb ├── postgresql │ └── import_test.rb ├── database.yml.sample ├── travis │ └── database.yml ├── support │ ├── generate.rb │ ├── factories.rb │ ├── active_support │ │ └── test_case_extensions.rb │ ├── assertions.rb │ ├── shared_examples │ │ └── on_duplicate_key_update.rb │ ├── mysql │ │ └── import_examples.rb │ └── postgresql │ │ └── import_examples.rb ├── value_sets_records_parser_test.rb ├── synchronize_test.rb ├── test_helper.rb ├── sqlite3 │ └── import_test.rb ├── value_sets_bytes_parser_test.rb └── import_test.rb ├── Brewfile ├── gemfiles ├── 3.1.gemfile ├── 3.2.gemfile ├── 4.0.gemfile ├── 4.1.gemfile ├── 4.2.gemfile └── 5.0.gemfile ├── benchmarks ├── models │ ├── test_innodb.rb │ ├── test_memory.rb │ └── test_myisam.rb ├── lib │ ├── float.rb │ ├── output_to_csv.rb │ ├── mysql2_benchmark.rb │ ├── output_to_html.rb │ ├── cli_parser.rb │ └── base.rb ├── schema │ └── mysql_schema.rb ├── README └── benchmark.rb ├── lib ├── activerecord-import │ ├── version.rb │ ├── adapters │ │ ├── mysql2_adapter.rb │ │ ├── em_mysql2_adapter.rb │ │ ├── sqlite3_adapter.rb │ │ ├── abstract_adapter.rb │ │ ├── mysql_adapter.rb │ │ └── postgresql_adapter.rb │ ├── active_record │ │ └── adapters │ │ │ ├── jdbcmysql_adapter.rb │ │ │ ├── mysql2_adapter.rb │ │ │ ├── sqlite3_adapter.rb │ │ │ ├── postgresql_adapter.rb │ │ │ ├── jdbcpostgresql_adapter.rb │ │ │ ├── abstract_adapter.rb │ │ │ └── seamless_database_pool_adapter.rb │ ├── mysql2.rb │ ├── sqlite3.rb │ ├── postgresql.rb │ ├── base.rb │ ├── value_sets_parser.rb │ ├── synchronize.rb │ └── import.rb └── activerecord-import.rb ├── .gitignore ├── .rubocop.yml ├── activerecord-import.gemspec ├── .travis.yml ├── Gemfile ├── .rubocop_todo.yml ├── Rakefile ├── CHANGELOG.md ├── LICENSE └── README.markdown /test/adapters/mysql2.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "mysql2" 2 | -------------------------------------------------------------------------------- /test/adapters/jdbcmysql.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "jdbcmysql" 2 | -------------------------------------------------------------------------------- /test/adapters/postgis.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "postgis" 2 | -------------------------------------------------------------------------------- /test/adapters/sqlite3.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "sqlite3" 2 | -------------------------------------------------------------------------------- /Brewfile: -------------------------------------------------------------------------------- 1 | brew "mysql" 2 | brew "postgresql" 3 | brew "sqlite" -------------------------------------------------------------------------------- /test/adapters/postgresql.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "postgresql" 2 | -------------------------------------------------------------------------------- /test/adapters/spatialite.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "spatialite" 2 | -------------------------------------------------------------------------------- /test/adapters/jdbcpostgresql.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "jdbcpostgresql" 2 | -------------------------------------------------------------------------------- /test/adapters/mysql2spatial.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "mysql2spatial" 2 | -------------------------------------------------------------------------------- /gemfiles/3.1.gemfile: -------------------------------------------------------------------------------- 1 | platforms :ruby do 2 | gem 'activerecord', '~> 3.1.0' 3 | end 4 | -------------------------------------------------------------------------------- /gemfiles/3.2.gemfile: -------------------------------------------------------------------------------- 1 | platforms :ruby do 2 | gem 'activerecord', '~> 3.2.0' 3 | end 4 | -------------------------------------------------------------------------------- /gemfiles/4.0.gemfile: -------------------------------------------------------------------------------- 1 | platforms :ruby do 2 | gem 'activerecord', '~> 4.0.0' 3 | end 4 | -------------------------------------------------------------------------------- /gemfiles/4.1.gemfile: -------------------------------------------------------------------------------- 1 | platforms :ruby do 2 | gem 'activerecord', '~> 4.1.0' 3 | end 4 | -------------------------------------------------------------------------------- /gemfiles/4.2.gemfile: -------------------------------------------------------------------------------- 1 | platforms :ruby do 2 | gem 'activerecord', '~> 4.2.0' 3 | end 4 | -------------------------------------------------------------------------------- /test/adapters/seamless_database_pool.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "seamless_database_pool" 2 | -------------------------------------------------------------------------------- /test/models/question.rb: -------------------------------------------------------------------------------- 1 | class Question < ActiveRecord::Base 2 | has_one :rule 3 | end 4 | -------------------------------------------------------------------------------- /test/models/rule.rb: -------------------------------------------------------------------------------- 1 | class Rule < ActiveRecord::Base 2 | belongs_to :question 3 | end 4 | -------------------------------------------------------------------------------- /gemfiles/5.0.gemfile: -------------------------------------------------------------------------------- 1 | platforms :ruby do 2 | gem 'activerecord', '~> 5.0.0.beta3' 3 | end 4 | -------------------------------------------------------------------------------- /test/models/group.rb: -------------------------------------------------------------------------------- 1 | class Group < ActiveRecord::Base 2 | self.table_name = 'group' 3 | end 4 | -------------------------------------------------------------------------------- /test/models/promotion.rb: -------------------------------------------------------------------------------- 1 | class Promotion < ActiveRecord::Base 2 | self.primary_key = :promotion_id 3 | end 4 | -------------------------------------------------------------------------------- /benchmarks/models/test_innodb.rb: -------------------------------------------------------------------------------- 1 | class TestInnoDb < ActiveRecord::Base 2 | self.table_name = 'test_innodb' 3 | end 4 | -------------------------------------------------------------------------------- /benchmarks/models/test_memory.rb: -------------------------------------------------------------------------------- 1 | class TestMemory < ActiveRecord::Base 2 | self.table_name = 'test_memory' 3 | end 4 | -------------------------------------------------------------------------------- /benchmarks/models/test_myisam.rb: -------------------------------------------------------------------------------- 1 | class TestMyISAM < ActiveRecord::Base 2 | self.table_name = 'test_myisam' 3 | end 4 | -------------------------------------------------------------------------------- /test/models/discount.rb: -------------------------------------------------------------------------------- 1 | class Discount < ActiveRecord::Base 2 | belongs_to :discountable, polymorphic: true 3 | end 4 | -------------------------------------------------------------------------------- /lib/activerecord-import/version.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord 2 | module Import 3 | VERSION = "0.13.0".freeze 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /test/models/chapter.rb: -------------------------------------------------------------------------------- 1 | class Chapter < ActiveRecord::Base 2 | belongs_to :book, inverse_of: :chapters 3 | validates :title, presence: true 4 | end 5 | -------------------------------------------------------------------------------- /test/models/end_note.rb: -------------------------------------------------------------------------------- 1 | class EndNote < ActiveRecord::Base 2 | belongs_to :book, inverse_of: :end_notes 3 | validates :note, presence: true 4 | end 5 | -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/mysql2_adapter.rb: -------------------------------------------------------------------------------- 1 | require "activerecord-import/adapters/mysql_adapter" 2 | 3 | module ActiveRecord::Import::Mysql2Adapter 4 | include ActiveRecord::Import::MysqlAdapter 5 | end 6 | -------------------------------------------------------------------------------- /test/models/widget.rb: -------------------------------------------------------------------------------- 1 | class Widget < ActiveRecord::Base 2 | self.primary_key = :w_id 3 | 4 | default_scope -> { where(active: true) } 5 | 6 | serialize :data, Hash 7 | serialize :json_data, JSON 8 | end 9 | -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/em_mysql2_adapter.rb: -------------------------------------------------------------------------------- 1 | require "activerecord-import/adapters/mysql_adapter" 2 | 3 | module ActiveRecord::Import::EMMysql2Adapter 4 | include ActiveRecord::Import::MysqlAdapter 5 | end 6 | -------------------------------------------------------------------------------- /test/postgis/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples') 3 | 4 | should_support_postgresql_import_functionality 5 | -------------------------------------------------------------------------------- /test/jdbcpostgresql/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples') 4 | 5 | should_support_postgresql_import_functionality 6 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/mysql_adapter" 2 | require "activerecord-import/adapters/mysql_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::MysqlAdapter 5 | include ActiveRecord::Import::MysqlAdapter 6 | end 7 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/mysql2_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/mysql2_adapter" 2 | require "activerecord-import/adapters/mysql2_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::Mysql2Adapter 5 | include ActiveRecord::Import::Mysql2Adapter 6 | end 7 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/sqlite3_adapter" 2 | require "activerecord-import/adapters/sqlite3_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::SQLite3Adapter 5 | include ActiveRecord::Import::SQLite3Adapter 6 | end 7 | -------------------------------------------------------------------------------- /test/models/book.rb: -------------------------------------------------------------------------------- 1 | class Book < ActiveRecord::Base 2 | belongs_to :topic, inverse_of: :books 3 | has_many :chapters, inverse_of: :book 4 | has_many :discounts, as: :discountable 5 | has_many :end_notes, inverse_of: :book 6 | enum status: [:draft, :published] if ENV['AR_VERSION'].to_f >= 4.1 7 | end 8 | -------------------------------------------------------------------------------- /test/mysql2/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | require File.expand_path(File.dirname(__FILE__) + '/../support/assertions') 3 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') 4 | 5 | should_support_mysql_import_functionality 6 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/postgresql_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/postgresql_adapter" 2 | require "activerecord-import/adapters/postgresql_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter 5 | include ActiveRecord::Import::PostgreSQLAdapter 6 | end 7 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/postgresql_adapter" 2 | require "activerecord-import/adapters/postgresql_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter 5 | include ActiveRecord::Import::PostgreSQLAdapter 6 | end 7 | -------------------------------------------------------------------------------- /test/jdbcmysql/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | require File.expand_path(File.dirname(__FILE__) + '/../support/assertions') 4 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') 5 | 6 | should_support_mysql_import_functionality 7 | -------------------------------------------------------------------------------- /test/mysqlspatial2/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | require File.expand_path(File.dirname(__FILE__) + '/../support/assertions') 4 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') 5 | 6 | should_support_mysql_import_functionality 7 | -------------------------------------------------------------------------------- /test/schema/version.rb: -------------------------------------------------------------------------------- 1 | class SchemaInfo < ActiveRecord::Base 2 | if respond_to?(:table_name=) 3 | self.table_name = 'schema_info' 4 | else 5 | # this is becoming deprecated in ActiveRecord but not all adapters supported it 6 | # at this time 7 | set_table_name 'schema_info' 8 | end 9 | VERSION = 12 10 | end 11 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/abstract_adapter.rb: -------------------------------------------------------------------------------- 1 | require "activerecord-import/adapters/abstract_adapter" 2 | 3 | module ActiveRecord # :nodoc: 4 | module ConnectionAdapters # :nodoc: 5 | class AbstractAdapter # :nodoc: 6 | include ActiveRecord::Import::AbstractAdapter::InstanceMethods 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/activerecord-import/mysql2.rb: -------------------------------------------------------------------------------- 1 | warn <<-MSG 2 | [DEPRECATION] loading activerecord-import via 'require "activerecord-import/"' 3 | is deprecated. Update to autorequire using 'require "activerecord-import"'. See 4 | http://github.com/zdennis/activerecord-import/wiki/Requiring for more information 5 | MSG 6 | 7 | require "activerecord-import" 8 | -------------------------------------------------------------------------------- /lib/activerecord-import/sqlite3.rb: -------------------------------------------------------------------------------- 1 | warn <<-MSG 2 | [DEPRECATION] loading activerecord-import via 'require "activerecord-import/"' 3 | is deprecated. Update to autorequire using 'require "activerecord-import"'. See 4 | http://github.com/zdennis/activerecord-import/wiki/Requiring for more information 5 | MSG 6 | 7 | require "activerecord-import" 8 | -------------------------------------------------------------------------------- /lib/activerecord-import/postgresql.rb: -------------------------------------------------------------------------------- 1 | warn <<-MSG 2 | [DEPRECATION] loading activerecord-import via 'require "activerecord-import/"' 3 | is deprecated. Update to autorequire using 'require "activerecord-import"'. See 4 | http://github.com/zdennis/activerecord-import/wiki/Requiring for more information 5 | MSG 6 | 7 | require "activerecord-import" 8 | -------------------------------------------------------------------------------- /benchmarks/lib/float.rb: -------------------------------------------------------------------------------- 1 | # Taken from http://www.programmingishard.com/posts/show/128 2 | # Posted by rbates 3 | class Float 4 | def round_to(x) 5 | (self * 10**x).round.to_f / 10**x 6 | end 7 | 8 | def ceil_to(x) 9 | (self * 10**x).ceil.to_f / 10**x 10 | end 11 | 12 | def floor_to(x) 13 | (self * 10**x).floor.to_f / 10**x 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb: -------------------------------------------------------------------------------- 1 | require "seamless_database_pool" 2 | require "active_record/connection_adapters/seamless_database_pool_adapter" 3 | require "activerecord-import/adapters/mysql_adapter" 4 | 5 | class ActiveRecord::ConnectionAdapters::SeamlessDatabasePoolAdapter 6 | include ActiveRecord::Import::MysqlAdapter 7 | end 8 | -------------------------------------------------------------------------------- /test/postgresql/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples') 3 | 4 | should_support_postgresql_import_functionality 5 | 6 | if ActiveRecord::Base.connection.supports_on_duplicate_key_update? 7 | should_support_postgresql_upsert_functionality 8 | end 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## MAC OS 2 | .DS_Store 3 | 4 | ## TEXTMATE 5 | *.tmproj 6 | tmtags 7 | 8 | ## EMACS 9 | *~ 10 | \#* 11 | .\#* 12 | 13 | ## VIM 14 | *.swp 15 | 16 | ## PROJECT::GENERAL 17 | coverage 18 | rdoc 19 | pkg 20 | *.gem 21 | *.lock 22 | 23 | ## PROJECT::SPECIFIC 24 | log/*.log 25 | test.db 26 | test/database.yml 27 | 28 | .ruby-* 29 | .bundle/ 30 | .redcar/ 31 | .rvmrc 32 | docsite/ 33 | -------------------------------------------------------------------------------- /test/models/topic.rb: -------------------------------------------------------------------------------- 1 | class Topic < ActiveRecord::Base 2 | validates_presence_of :author_name 3 | validates :title, numericality: { only_integer: true }, on: :context_test 4 | 5 | has_many :books, inverse_of: :topic 6 | belongs_to :parent, class_name: "Topic" 7 | 8 | composed_of :description, mapping: [%w(title title), %w(author_name author_name)], allow_nil: true, class_name: "TopicDescription" 9 | end 10 | -------------------------------------------------------------------------------- /benchmarks/lib/output_to_csv.rb: -------------------------------------------------------------------------------- 1 | require 'csv' 2 | 3 | module OutputToCSV 4 | def self.output_results( filename, results ) 5 | CSV.open( filename, 'w' ) do |csv| 6 | # Iterate over each result set, which contains many results 7 | results.each do |result_set| 8 | columns = [] 9 | times = [] 10 | result_set.each do |result| 11 | columns << result.description 12 | times << result.tms.real 13 | end 14 | csv << columns 15 | csv << times 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /benchmarks/schema/mysql_schema.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Schema.define do 2 | create_table :test_myisam, options: 'ENGINE=MyISAM', force: true do |t| 3 | t.column :my_name, :string, null: false 4 | t.column :description, :string 5 | end 6 | 7 | create_table :test_innodb, options: 'ENGINE=InnoDb', force: true do |t| 8 | t.column :my_name, :string, null: false 9 | t.column :description, :string 10 | end 11 | 12 | create_table :test_memory, options: 'ENGINE=Memory', force: true do |t| 13 | t.column :my_name, :string, null: false 14 | t.column :description, :string 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/activerecord-import.rb: -------------------------------------------------------------------------------- 1 | # rubocop:disable Style/FileName 2 | 3 | ActiveSupport.on_load(:active_record) do 4 | class ActiveRecord::Base 5 | class << self 6 | def establish_connection_with_activerecord_import(*args) 7 | conn = establish_connection_without_activerecord_import(*args) 8 | if !ActiveRecord.const_defined?(:Import) || !ActiveRecord::Import.respond_to?(:load_from_connection_pool) 9 | require "activerecord-import/base" 10 | end 11 | 12 | ActiveRecord::Import.load_from_connection_pool connection_pool 13 | conn 14 | end 15 | alias_method_chain :establish_connection, :activerecord_import 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /benchmarks/lib/mysql2_benchmark.rb: -------------------------------------------------------------------------------- 1 | class Mysql2Benchmark < BenchmarkBase 2 | def benchmark_all( array_of_cols_and_vals ) 3 | methods = self.methods.find_all { |m| m =~ /benchmark_/ } 4 | methods.delete_if { |m| m =~ /benchmark_(all|model)/ } 5 | methods.each { |method| send( method, array_of_cols_and_vals ) } 6 | end 7 | 8 | def benchmark_myisam( array_of_cols_and_vals ) 9 | bm_model( TestMyISAM, array_of_cols_and_vals ) 10 | end 11 | 12 | def benchmark_innodb( array_of_cols_and_vals ) 13 | bm_model( TestInnoDb, array_of_cols_and_vals ) 14 | end 15 | 16 | def benchmark_memory( array_of_cols_and_vals ) 17 | bm_model( TestMemory, array_of_cols_and_vals ) 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /test/schema/mysql_schema.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Schema.define do 2 | create_table :books, options: 'ENGINE=MyISAM', force: true do |t| 3 | t.column :title, :string, null: false 4 | t.column :publisher, :string, null: false, default: 'Default Publisher' 5 | t.column :author_name, :string, null: false 6 | t.column :created_at, :datetime 7 | t.column :created_on, :datetime 8 | t.column :updated_at, :datetime 9 | t.column :updated_on, :datetime 10 | t.column :publish_date, :date 11 | t.column :topic_id, :integer 12 | t.column :for_sale, :boolean, default: true 13 | t.column :status, :integer 14 | end 15 | execute "ALTER TABLE books ADD FULLTEXT( `title`, `publisher`, `author_name` )" 16 | end 17 | -------------------------------------------------------------------------------- /test/database.yml.sample: -------------------------------------------------------------------------------- 1 | common: &common 2 | username: root 3 | password: 4 | encoding: utf8 5 | host: localhost 6 | database: activerecord_import_test 7 | 8 | mysql2: &mysql2 9 | <<: *common 10 | adapter: mysql2 11 | 12 | mysql2spatial: 13 | <<: *mysql2 14 | 15 | seamless_database_pool: 16 | <<: *common 17 | adapter: seamless_database_pool 18 | pool_adapter: mysql2 19 | master: 20 | host: localhost 21 | 22 | postgresql: &postgresql 23 | <<: *common 24 | username: postgres 25 | adapter: postgresql 26 | min_messages: warning 27 | 28 | postgis: 29 | <<: *postgresql 30 | 31 | oracle: 32 | <<: *common 33 | adapter: oracle 34 | min_messages: debug 35 | 36 | sqlite: 37 | adapter: sqlite 38 | dbfile: test.db 39 | 40 | sqlite3: &sqlite3 41 | adapter: sqlite3 42 | database: test.db 43 | 44 | spatialite: 45 | <<: *sqlite3 46 | -------------------------------------------------------------------------------- /test/travis/database.yml: -------------------------------------------------------------------------------- 1 | common: &common 2 | username: root 3 | password: 4 | encoding: utf8 5 | host: localhost 6 | database: activerecord_import_test 7 | 8 | mysql2: &mysql2 9 | <<: *common 10 | adapter: mysql2 11 | 12 | mysql2spatial: 13 | <<: *mysql2 14 | 15 | seamless_database_pool: 16 | <<: *common 17 | adapter: seamless_database_pool 18 | pool_adapter: mysql2 19 | master: 20 | host: localhost 21 | 22 | postgresql: &postgresql 23 | <<: *common 24 | username: postgres 25 | adapter: postgresql 26 | min_messages: warning 27 | 28 | postgis: 29 | <<: *postgresql 30 | 31 | oracle: 32 | <<: *common 33 | adapter: oracle 34 | min_messages: debug 35 | 36 | sqlite: 37 | adapter: sqlite 38 | dbfile: test.db 39 | 40 | sqlite3: &sqlite3 41 | adapter: sqlite3 42 | database: ":memory:" 43 | 44 | spatialite: 45 | <<: *sqlite3 46 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_from: .rubocop_todo.yml 2 | 3 | Lint/EndAlignment: 4 | AlignWith: variable 5 | 6 | Metrics/AbcSize: 7 | Enabled: false 8 | 9 | Metrics/ClassLength: 10 | Enabled: false 11 | 12 | Metrics/CyclomaticComplexity: 13 | Enabled: false 14 | 15 | Metrics/LineLength: 16 | Enabled: false 17 | 18 | Metrics/MethodLength: 19 | Enabled: false 20 | 21 | Metrics/ModuleLength: 22 | Enabled: false 23 | 24 | Metrics/PerceivedComplexity: 25 | Enabled: false 26 | 27 | Style/AlignParameters: 28 | EnforcedStyle: with_fixed_indentation 29 | 30 | Style/ClassAndModuleChildren: 31 | Enabled: false 32 | 33 | Style/Documentation: 34 | Enabled: false 35 | 36 | Style/ElseAlignment: 37 | Enabled: false 38 | 39 | Style/SpaceInsideParens: 40 | Enabled: false 41 | 42 | Style/SpecialGlobalVars: 43 | Enabled: false 44 | 45 | Style/StringLiterals: 46 | Enabled: false 47 | 48 | Style/TrailingCommaInLiteral: 49 | Enabled: false 50 | -------------------------------------------------------------------------------- /test/support/generate.rb: -------------------------------------------------------------------------------- 1 | class ActiveSupport::TestCase 2 | def Build(*args) # rubocop:disable Style/MethodName 3 | n = args.shift if args.first.is_a?(Numeric) 4 | factory = args.shift 5 | factory_girl_args = args.shift || {} 6 | 7 | if n 8 | [].tap do |collection| 9 | n.times.each { collection << FactoryGirl.build(factory.to_s.singularize.to_sym, factory_girl_args) } 10 | end 11 | else 12 | FactoryGirl.build(factory.to_s.singularize.to_sym, factory_girl_args) 13 | end 14 | end 15 | 16 | def Generate(*args) # rubocop:disable Style/MethodName 17 | n = args.shift if args.first.is_a?(Numeric) 18 | factory = args.shift 19 | factory_girl_args = args.shift || {} 20 | 21 | if n 22 | [].tap do |collection| 23 | n.times.each { collection << FactoryGirl.create(factory.to_s.singularize.to_sym, factory_girl_args) } 24 | end 25 | else 26 | FactoryGirl.create(factory.to_s.singularize.to_sym, factory_girl_args) 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /activerecord-import.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | require File.expand_path('../lib/activerecord-import/version', __FILE__) 3 | 4 | Gem::Specification.new do |gem| 5 | gem.authors = ["Zach Dennis"] 6 | gem.email = ["zach.dennis@gmail.com"] 7 | gem.summary = "Bulk-loading extension for ActiveRecord" 8 | gem.description = "Extraction of the ActiveRecord::Base#import functionality from ar-extensions for Rails 3 and beyond" 9 | gem.homepage = "http://github.com/zdennis/activerecord-import" 10 | gem.license = "Ruby" 11 | 12 | gem.files = `git ls-files`.split($\) 13 | gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) } 14 | gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) 15 | gem.name = "activerecord-import" 16 | gem.require_paths = ["lib"] 17 | gem.version = ActiveRecord::Import::VERSION 18 | 19 | gem.required_ruby_version = ">= 1.9.2" 20 | 21 | gem.add_runtime_dependency "activerecord", ">= 3.0" 22 | gem.add_development_dependency "rake" 23 | end 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | cache: bundler 3 | rvm: 4 | - 2.2.4 5 | 6 | env: 7 | global: 8 | # https://github.com/discourse/discourse/blob/master/.travis.yml 9 | - RUBY_GC_MALLOC_LIMIT=50000000 10 | matrix: 11 | - AR_VERSION=3.1 12 | - AR_VERSION=3.2 13 | - AR_VERSION=4.0 14 | - AR_VERSION=4.1 15 | - AR_VERSION=4.2 16 | - AR_VERSION=5.0 17 | 18 | matrix: 19 | fast_finish: true 20 | 21 | before_script: 22 | - mysql -e 'create database activerecord_import_test;' 23 | - psql -c 'create database activerecord_import_test;' -U postgres 24 | - psql -U postgres -c "create extension postgis" 25 | - cp test/travis/database.yml test/database.yml 26 | 27 | addons: 28 | apt: 29 | sources: 30 | - travis-ci/sqlite3 31 | packages: 32 | - sqlite3 33 | 34 | script: 35 | - bundle exec rake test:mysql2 36 | - bundle exec rake test:mysql2spatial 37 | - bundle exec rake test:postgis 38 | - bundle exec rake test:postgresql 39 | - bundle exec rake test:seamless_database_pool 40 | - bundle exec rake test:spatialite 41 | - bundle exec rake test:sqlite3 42 | - bundle exec rubocop 43 | 44 | sudo: false 45 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | group :development, :test do 6 | gem 'rubocop', '~> 0.38.0' 7 | end 8 | 9 | # Database Adapters 10 | platforms :ruby do 11 | gem "mysql2", "~> 0.3.0" 12 | gem "pg", "~> 0.9" 13 | gem "sqlite3", "~> 1.3.10" 14 | gem "seamless_database_pool", "~> 1.0.13" 15 | end 16 | 17 | platforms :jruby do 18 | gem "jdbc-mysql" 19 | gem "jdbc-postgres" 20 | gem "activerecord-jdbcmysql-adapter" 21 | gem "activerecord-jdbcpostgresql-adapter" 22 | end 23 | 24 | # Support libs 25 | gem "factory_girl", "~> 4.2.0" 26 | gem "timecop" 27 | gem "chronic" 28 | 29 | # Debugging 30 | platforms :jruby do 31 | gem "ruby-debug-base", "= 0.10.4" 32 | end 33 | 34 | platforms :jruby, :mri_18 do 35 | gem "ruby-debug", "= 0.10.4" 36 | end 37 | 38 | platforms :mri_19 do 39 | gem "debugger" 40 | end 41 | 42 | platforms :ruby do 43 | gem "pry-byebug" 44 | end 45 | 46 | version = ENV['AR_VERSION'] || "4.2" 47 | 48 | if version >= "4.0" 49 | gem "minitest" 50 | else 51 | gem "test-unit" 52 | end 53 | 54 | eval_gemfile File.expand_path("../gemfiles/#{version}.gemfile", __FILE__) 55 | -------------------------------------------------------------------------------- /lib/activerecord-import/base.rb: -------------------------------------------------------------------------------- 1 | require "pathname" 2 | require "active_record" 3 | require "active_record/version" 4 | 5 | module ActiveRecord::Import 6 | ADAPTER_PATH = "activerecord-import/active_record/adapters".freeze 7 | 8 | def self.base_adapter(adapter) 9 | case adapter 10 | when 'mysql2spatial' then 'mysql2' 11 | when 'spatialite' then 'sqlite3' 12 | when 'postgis' then 'postgresql' 13 | else adapter 14 | end 15 | end 16 | 17 | # Loads the import functionality for a specific database adapter 18 | def self.require_adapter(adapter) 19 | require File.join(ADAPTER_PATH, "/abstract_adapter") 20 | begin 21 | require File.join(ADAPTER_PATH, "/#{base_adapter(adapter)}_adapter") 22 | rescue LoadError 23 | # fallback 24 | end 25 | end 26 | 27 | # Loads the import functionality for the passed in ActiveRecord connection 28 | def self.load_from_connection_pool(connection_pool) 29 | require_adapter connection_pool.spec.config[:adapter] 30 | end 31 | end 32 | 33 | require 'activerecord-import/import' 34 | require 'activerecord-import/active_record/adapters/abstract_adapter' 35 | require 'activerecord-import/synchronize' 36 | require 'activerecord-import/value_sets_parser' 37 | -------------------------------------------------------------------------------- /test/value_sets_records_parser_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/test_helper') 2 | 3 | require 'activerecord-import/value_sets_parser' 4 | 5 | describe "ActiveRecord::Import::ValueSetsRecordsParser" do 6 | context "#parse - computing insert value sets" do 7 | let(:parser) { ActiveRecord::Import::ValueSetsRecordsParser } 8 | let(:base_sql) { "INSERT INTO atable (a,b,c)" } 9 | let(:values) { ["(1,2,3)", "(2,3,4)", "(3,4,5)"] } 10 | 11 | context "when the max number of records is 1" do 12 | it "should return 3 value sets when given 3 values sets" do 13 | value_sets = parser.parse values, max_records: 1 14 | assert_equal 3, value_sets.size 15 | end 16 | end 17 | 18 | context "when the max number of records is 2" do 19 | it "should return 2 value sets when given 3 values sets" do 20 | value_sets = parser.parse values, max_records: 2 21 | assert_equal 2, value_sets.size 22 | end 23 | end 24 | 25 | context "when the max number of records is 3" do 26 | it "should return 1 value sets when given 3 values sets" do 27 | value_sets = parser.parse values, max_records: 3 28 | assert_equal 1, value_sets.size 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /benchmarks/README: -------------------------------------------------------------------------------- 1 | To run the benchmarks, from within the benchmarks run: 2 | ruby benchmark.rb [options] 3 | 4 | The following options are supported: 5 | --adapter [String] The database adapter to use. IE: mysql, postgresql, oracle 6 | 7 | --do-not-delete By default all records in the benchmark tables will be deleted at the end of the benchmark. This flag indicates not to delete the benchmark data. 8 | --num [Integer] The number of objects to benchmark. (Required!) 9 | --table-type [String] The table type to test. This can be used multiple times. By default it is all table types. 10 | --to-csv [String] Print results in a CSV file format 11 | --to-html [String] Print results in HTML format (String filename must be supplied) 12 | 13 | See "ruby benchmark.rb -h" for the complete listing of options. 14 | 15 | EXAMPLES 16 | -------- 17 | To output to html format: 18 | ruby benchmark.rb --adapter=mysql --to-html=results.html 19 | 20 | To output to csv format: 21 | ruby benchmark.rb --adapter=mysql --to-csv=results.csv 22 | 23 | LIMITATIONS 24 | ----------- 25 | Currently MySQL is the only supported adapter to benchmark. 26 | 27 | AUTHOR 28 | ------ 29 | Zach Dennis 30 | zach.dennis@gmail.com 31 | http://www.continuousthinking.com 32 | 33 | -------------------------------------------------------------------------------- /.rubocop_todo.yml: -------------------------------------------------------------------------------- 1 | # This configuration was generated by 2 | # `rubocop --auto-gen-config` 3 | # on 2016-03-17 18:14:55 -0700 using RuboCop version 0.38.0. 4 | # The point is for the user to remove these configuration records 5 | # one by one as the offenses are removed from the code base. 6 | # Note that changes in the inspected code, or installation of new 7 | # versions of RuboCop, may require this file to be generated again. 8 | 9 | # Offense count: 2 10 | Lint/HandleExceptions: 11 | Exclude: 12 | - 'lib/activerecord-import/base.rb' 13 | - 'test/import_test.rb' 14 | 15 | # Offense count: 2 16 | Lint/RescueException: 17 | Exclude: 18 | - 'benchmarks/lib/cli_parser.rb' 19 | - 'test/import_test.rb' 20 | 21 | # Offense count: 4 22 | # Cop supports --auto-correct. 23 | # Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods. 24 | Lint/UnusedMethodArgument: 25 | Exclude: 26 | - 'lib/activerecord-import/adapters/postgresql_adapter.rb' 27 | - 'lib/activerecord-import/import.rb' 28 | 29 | # Offense count: 2 30 | # Cop supports --auto-correct. 31 | # Configuration parameters: Keywords. 32 | # Keywords: TODO, FIXME, OPTIMIZE, HACK, REVIEW 33 | Style/CommentAnnotation: 34 | Exclude: 35 | - 'benchmarks/lib/cli_parser.rb' 36 | - 'lib/activerecord-import/import.rb' 37 | -------------------------------------------------------------------------------- /test/support/factories.rb: -------------------------------------------------------------------------------- 1 | FactoryGirl.define do 2 | sequence(:book_title) { |n| "Book #{n}" } 3 | sequence(:chapter_title) { |n| "Chapter #{n}" } 4 | sequence(:end_note) { |n| "Endnote #{n}" } 5 | 6 | factory :group do 7 | sequence(:order) { |n| "Order #{n}" } 8 | end 9 | 10 | factory :invalid_topic, class: "Topic" do 11 | sequence(:title) { |n| "Title #{n}" } 12 | author_name nil 13 | end 14 | 15 | factory :topic do 16 | sequence(:title) { |n| "Title #{n}" } 17 | sequence(:author_name) { |n| "Author #{n}" } 18 | end 19 | 20 | factory :widget do 21 | sequence(:w_id) { |n| n } 22 | end 23 | 24 | factory :question do 25 | sequence(:body) { |n| "Text #{n}" } 26 | 27 | trait :with_rule do 28 | after(:build) do |question| 29 | question.build_rule(FactoryGirl.attributes_for(:rule)) 30 | end 31 | end 32 | end 33 | 34 | factory :rule do 35 | sequence(:condition_text) { |n| "q_#{n}_#{n}" } 36 | end 37 | 38 | factory :topic_with_book, parent: :topic do 39 | after(:build) do |topic| 40 | 2.times do 41 | book = topic.books.build(title: FactoryGirl.generate(:book_title), author_name: 'Stephen King') 42 | 3.times do 43 | book.chapters.build(title: FactoryGirl.generate(:chapter_title)) 44 | end 45 | 46 | 4.times do 47 | book.end_notes.build(note: FactoryGirl.generate(:end_note)) 48 | end 49 | end 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/sqlite3_adapter.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord::Import::SQLite3Adapter 2 | include ActiveRecord::Import::ImportSupport 3 | 4 | MIN_VERSION_FOR_IMPORT = "3.7.11".freeze 5 | SQLITE_LIMIT_COMPOUND_SELECT = 500 6 | 7 | # Override our conformance to ActiveRecord::Import::ImportSupport interface 8 | # to ensure that we only support import in supported version of SQLite. 9 | # Which INSERT statements with multiple value sets was introduced in 3.7.11. 10 | def supports_import?(current_version = sqlite_version) 11 | if current_version >= MIN_VERSION_FOR_IMPORT 12 | true 13 | else 14 | false 15 | end 16 | end 17 | 18 | # +sql+ can be a single string or an array. If it is an array all 19 | # elements that are in position >= 1 will be appended to the final SQL. 20 | def insert_many(sql, values, *args) # :nodoc: 21 | number_of_inserts = 0 22 | base_sql, post_sql = if sql.is_a?( String ) 23 | [sql, ''] 24 | elsif sql.is_a?( Array ) 25 | [sql.shift, sql.join( ' ' )] 26 | end 27 | 28 | value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values, 29 | max_records: SQLITE_LIMIT_COMPOUND_SELECT) 30 | 31 | value_sets.each do |value_set| 32 | number_of_inserts += 1 33 | sql2insert = base_sql + value_set.join( ',' ) + post_sql 34 | insert( sql2insert, *args ) 35 | end 36 | 37 | [number_of_inserts, []] 38 | end 39 | 40 | def next_value_for_sequence(sequence_name) 41 | %{nextval('#{sequence_name}')} 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/activerecord-import/value_sets_parser.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord::Import 2 | class ValueSetsBytesParser 3 | attr_reader :reserved_bytes, :max_bytes, :values 4 | 5 | def self.parse(values, options) 6 | new(values, options).parse 7 | end 8 | 9 | def initialize(values, options) 10 | @values = values 11 | @reserved_bytes = options[:reserved_bytes] 12 | @max_bytes = options[:max_bytes] 13 | end 14 | 15 | def parse 16 | value_sets = [] 17 | arr = [] 18 | current_size = 0 19 | values.each_with_index do |val, i| 20 | comma_bytes = arr.size 21 | bytes_thus_far = reserved_bytes + current_size + val.bytesize + comma_bytes 22 | if bytes_thus_far <= max_bytes 23 | current_size += val.bytesize 24 | arr << val 25 | else 26 | value_sets << arr 27 | arr = [val] 28 | current_size = val.bytesize 29 | end 30 | 31 | # if we're on the last iteration push whatever we have in arr to value_sets 32 | value_sets << arr if i == (values.size - 1) 33 | end 34 | 35 | [*value_sets] 36 | end 37 | end 38 | 39 | class ValueSetsRecordsParser 40 | attr_reader :max_records, :values 41 | 42 | def self.parse(values, options) 43 | new(values, options).parse 44 | end 45 | 46 | def initialize(values, options) 47 | @values = values 48 | @max_records = options[:max_records] 49 | end 50 | 51 | def parse 52 | @values.in_groups_of(max_records, false) 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /test/synchronize_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../test_helper', __FILE__) 2 | 3 | describe ".synchronize" do 4 | let(:topics) { Generate(3, :topics) } 5 | let(:titles) { %w(one two three) } 6 | 7 | setup do 8 | # update records outside of ActiveRecord knowing about it 9 | Topic.connection.execute( "UPDATE #{Topic.table_name} SET title='#{titles[0]}_haha' WHERE id=#{topics[0].id}", "Updating record 1 without ActiveRecord" ) 10 | Topic.connection.execute( "UPDATE #{Topic.table_name} SET title='#{titles[1]}_haha' WHERE id=#{topics[1].id}", "Updating record 2 without ActiveRecord" ) 11 | Topic.connection.execute( "UPDATE #{Topic.table_name} SET title='#{titles[2]}_haha' WHERE id=#{topics[2].id}", "Updating record 3 without ActiveRecord" ) 12 | end 13 | 14 | it "reloads data for the specified records" do 15 | Topic.synchronize topics 16 | 17 | actual_titles = topics.map(&:title) 18 | assert_equal "#{titles[0]}_haha", actual_titles[0], "the first record was not correctly updated" 19 | assert_equal "#{titles[1]}_haha", actual_titles[1], "the second record was not correctly updated" 20 | assert_equal "#{titles[2]}_haha", actual_titles[2], "the third record was not correctly updated" 21 | end 22 | 23 | it "the synchronized records aren't dirty" do 24 | # Update the in memory records so they're dirty 25 | topics.each { |topic| topic.title = 'dirty title' } 26 | 27 | Topic.synchronize topics 28 | 29 | assert_equal false, topics[0].changed?, "the first record was dirty" 30 | assert_equal false, topics[1].changed?, "the second record was dirty" 31 | assert_equal false, topics[2].changed?, "the third record was dirty" 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler" 2 | Bundler.setup 3 | 4 | require 'rake' 5 | require 'rake/testtask' 6 | 7 | namespace :display do 8 | task :notice do 9 | puts 10 | puts "To run tests you must supply the adapter, see rake -T for more information." 11 | puts 12 | end 13 | end 14 | task default: ["display:notice"] 15 | 16 | ADAPTERS = %w(mysql2 jdbcmysql jdbcpostgresql postgresql sqlite3 seamless_database_pool mysql2spatial spatialite postgis).freeze 17 | ADAPTERS.each do |adapter| 18 | namespace :test do 19 | desc "Runs #{adapter} database tests." 20 | Rake::TestTask.new(adapter) do |t| 21 | # FactoryGirl has an issue with warnings, so turn off, so noisy 22 | # t.warning = true 23 | t.test_files = FileList["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/active_record/*_test.rb", "test/#{adapter}/**/*_test.rb"] 24 | end 25 | task adapter 26 | end 27 | end 28 | 29 | begin 30 | require 'rcov/rcovtask' 31 | adapter = ENV['ARE_DB'] 32 | Rcov::RcovTask.new do |test| 33 | test.libs << 'test' 34 | test.pattern = ["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/#{adapter}/**/*_test.rb"] 35 | test.verbose = true 36 | end 37 | rescue LoadError 38 | task :rcov do 39 | abort "RCov is not available. In order to run rcov, you must: sudo gem install rcov" 40 | end 41 | end 42 | 43 | require 'rdoc/task' 44 | Rake::RDocTask.new do |rdoc| 45 | version = File.exist?('VERSION') ? File.read('VERSION') : "" 46 | 47 | rdoc.rdoc_dir = 'rdoc' 48 | rdoc.title = "activerecord-import #{version}" 49 | rdoc.rdoc_files.include('README*') 50 | rdoc.rdoc_files.include('lib/**/*.rb') 51 | end 52 | 53 | require 'rubocop/rake_task' 54 | RuboCop::RakeTask.new 55 | -------------------------------------------------------------------------------- /benchmarks/lib/output_to_html.rb: -------------------------------------------------------------------------------- 1 | require 'erb' 2 | 3 | module OutputToHTML 4 | TEMPLATE_HEADER = <<"EOT".freeze 5 |
6 | All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed 7 | before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against 8 | the very first column (which is always the default ActiveRecord::Base.create method. 9 |
10 | EOT 11 | 12 | TEMPLATE = <<"EOT".freeze 13 | 27 | 28 | 29 | <% columns.each do |col| %> 30 | 31 | <% end %> 32 | 33 | 34 | <% times.each do |time| %> 35 | 36 | <% end %> 37 | 38 | 39 |
<%= col %>
<%= time %>
 
40 | EOT 41 | 42 | def self.output_results( filename, results ) 43 | html = '' 44 | results.each do |result_set| 45 | columns = [] 46 | times = [] 47 | result_set.each do |result| 48 | columns << result.description 49 | if result.failed 50 | times << "failed" 51 | else 52 | time = result.tms.real.round_to( 3 ) 53 | speedup = ( result_set.first.tms.real / result.tms.real ).round 54 | times << (result == result_set.first ? time.to_s : "#{time} (#{speedup}x speedup)") 55 | end 56 | end 57 | 58 | template = ERB.new( TEMPLATE, 0, "%<>") 59 | html << template.result( binding ) 60 | end 61 | 62 | File.open( filename, 'w' ) { |file| file.write( TEMPLATE_HEADER + html ) } 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require 'pathname' 2 | test_dir = Pathname.new File.dirname(__FILE__) 3 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 4 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 5 | 6 | require "fileutils" 7 | 8 | ENV["RAILS_ENV"] = "test" 9 | 10 | require "bundler" 11 | Bundler.setup 12 | 13 | require 'pry' unless RbConfig::CONFIG["RUBY_INSTALL_NAME"] =~ /jruby/ 14 | 15 | require "active_record" 16 | require "active_record/fixtures" 17 | require "active_support/test_case" 18 | 19 | if ActiveSupport::VERSION::STRING < "4.0" 20 | require 'test/unit' 21 | else 22 | require 'active_support/testing/autorun' 23 | end 24 | 25 | require 'timecop' 26 | require 'chronic' 27 | 28 | require "ruby-debug" if RUBY_VERSION.to_f < 1.9 29 | 30 | adapter = ENV["ARE_DB"] || "sqlite3" 31 | 32 | FileUtils.mkdir_p 'log' 33 | ActiveRecord::Base.logger = Logger.new("log/test.log") 34 | ActiveRecord::Base.logger.level = Logger::DEBUG 35 | ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter] 36 | ActiveRecord::Base.default_timezone = :utc 37 | 38 | require "activerecord-import" 39 | ActiveRecord::Base.establish_connection :test 40 | 41 | ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh| 42 | ActiveRecord::Base.logger.info hsh[:sql] 43 | end 44 | 45 | require "factory_girl" 46 | Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each { |file| require file } 47 | 48 | # Load base/generic schema 49 | require test_dir.join("schema/version") 50 | require test_dir.join("schema/generic_schema") 51 | adapter_schema = test_dir.join("schema/#{adapter}_schema.rb") 52 | require adapter_schema if File.exist?(adapter_schema) 53 | 54 | Dir[File.dirname(__FILE__) + "/models/*.rb"].each { |file| require file } 55 | 56 | # Prevent this deprecation warning from breaking the tests. 57 | Rake::FileList.send(:remove_method, :import) 58 | 59 | ActiveSupport::TestCase.test_order = :random if ENV['AR_VERSION'].to_f >= 4.2 60 | -------------------------------------------------------------------------------- /test/support/active_support/test_case_extensions.rb: -------------------------------------------------------------------------------- 1 | class ActiveSupport::TestCase 2 | include ActiveRecord::TestFixtures 3 | self.use_transactional_fixtures = true 4 | 5 | class << self 6 | def requires_active_record_version(version_string, &blk) 7 | return unless Gem::Dependency.new('', version_string).match?('', ActiveRecord::VERSION::STRING) 8 | instance_eval(&blk) 9 | end 10 | 11 | def assertion(name, &block) 12 | mc = class << self; self; end 13 | mc.class_eval do 14 | define_method(name) do 15 | it(name, &block) 16 | end 17 | end 18 | end 19 | 20 | def asssertion_group(name, &block) 21 | mc = class << self; self; end 22 | mc.class_eval do 23 | define_method(name, &block) 24 | end 25 | end 26 | 27 | def macro(name, &block) 28 | class_eval do 29 | define_method(name, &block) 30 | end 31 | end 32 | 33 | def describe(description, toplevel = nil, &blk) 34 | text = toplevel ? description : "#{name} #{description}" 35 | klass = Class.new(self) 36 | 37 | klass.class_eval <<-RUBY_EVAL 38 | def self.name 39 | "#{text}" 40 | end 41 | RUBY_EVAL 42 | 43 | # do not inherit test methods from the superclass 44 | klass.class_eval do 45 | instance_methods.grep(/^test.+/) do |method| 46 | undef_method method 47 | end 48 | end 49 | 50 | klass.instance_eval(&blk) 51 | end 52 | alias context describe 53 | 54 | def let(name, &blk) 55 | define_method(name) do 56 | instance_variable_name = "@__let_#{name}" 57 | return instance_variable_get(instance_variable_name) if instance_variable_defined?(instance_variable_name) 58 | instance_variable_set(instance_variable_name, instance_eval(&blk)) 59 | end 60 | end 61 | 62 | def it(description, &blk) 63 | define_method("test_#{name}_#{description}", &blk) 64 | end 65 | end 66 | end 67 | 68 | def describe(description, &blk) 69 | ActiveSupport::TestCase.describe(description, true, &blk) 70 | end 71 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Changes in 0.13.0 2 | 3 | ### New Features 4 | 5 | * Addition of :batch_size option to control the number of rows to insert per INSERT statement. The default is the total number of records being inserted so there is a single INSERT statement. Thanks to @jkowens via \#245 6 | 7 | * Addition `import!` which will raise an exception if a validation occurs. It will fail fast. Thanks to @jkowens via \#246 8 | 9 | ### Fixes 10 | 11 | * Fixing issue with recursive import when utilizing the `:on_duplicate_key_update` option. The `on_duplicate_key_update` only applies to parent models at this time. Thanks to @yuri-karpovich for reporting and @jkowens for fixing via \#249 12 | 13 | ### Misc 14 | 15 | * Refactoring of fetching and assigning attributes. Thanks to @jkownes via \#259 16 | * Lots of code cleanup and addition of Rubocop linter. Thanks to @sferik via \#256 and \#250 17 | * Resolving errors with the test suite when running against ActiveRecord 4.0 and 4.1. Thanks to @jkowens via \#262 18 | * Cleaning up the TravisCI settings and packages. Thanks to @sferik via \#258 and \#251 19 | 20 | ## Changes in 0.12.0 21 | 22 | ### New Features 23 | 24 | * PostgreSQL UPSERT support has been added. Thanks @jkowens via \#218 25 | 26 | ### Fixes 27 | 28 | * has_one and has_many associations will now be recursively imported regardless of :autosave being set. Thanks @sferik, @jkowens via \#243, \#234 29 | * Fixing an issue with enum column support for Rails > 4.1. Thanks @aquajach via \#235 30 | 31 | ### Removals 32 | 33 | * Support for em-synchrony has been removed since it appears the project has been abandoned. Thanks @sferik, @zdennis via \#239 34 | * Support for the mysql gem/adapter has been removed since it has officially been abandoned. Use the mysql2 gem/adapter instead. Thanks @sferik, @zdennis via \#239 35 | 36 | ### Misc 37 | 38 | * Cleaned up TravisCI output and removing deprecation warnings. Thanks @jkowens, @zdennis \#242 39 | 40 | 41 | ## Changes before 0.12.0 42 | 43 | > Never look back. What's gone is now history. But in the process make memory of events to help you understand what will help you to make your dream a true story. Mistakes of the past are lessons, success of the past is inspiration. – Dr. Anil Kr Sinha 44 | -------------------------------------------------------------------------------- /benchmarks/benchmark.rb: -------------------------------------------------------------------------------- 1 | require 'pathname' 2 | require "fileutils" 3 | require "active_record" 4 | 5 | benchmark_dir = File.dirname(__FILE__) 6 | 7 | $LOAD_PATH.unshift('.') 8 | 9 | # Get the gem into the load path 10 | $LOAD_PATH.unshift(File.join(benchmark_dir, '..', 'lib')) 11 | 12 | # Load the benchmark files 13 | Dir[File.join( benchmark_dir, 'lib', '*.rb' )].sort.each { |f| require f } 14 | 15 | # Parse the options passed in via the command line 16 | options = BenchmarkOptionParser.parse( ARGV ) 17 | 18 | FileUtils.mkdir_p 'log' 19 | ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter] 20 | ActiveRecord::Base.logger = Logger.new("log/test.log") 21 | ActiveRecord::Base.logger.level = Logger::DEBUG 22 | ActiveRecord::Base.default_timezone = :utc 23 | 24 | require "activerecord-import" 25 | ActiveRecord::Base.establish_connection(:test) 26 | 27 | ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh| 28 | ActiveRecord::Base.logger.info hsh[:sql] 29 | end 30 | 31 | # Load base/generic schema 32 | require File.join(benchmark_dir, "../test/schema/version") 33 | require File.join(benchmark_dir, "../test/schema/generic_schema") 34 | adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb") 35 | require adapter_schema if File.exist?(adapter_schema) 36 | 37 | Dir[File.dirname(__FILE__) + "/models/*.rb"].each { |file| require file } 38 | 39 | require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" ) 40 | 41 | table_types = nil 42 | table_types = if options.benchmark_all_types 43 | ["all"] 44 | else 45 | options.table_types.keys 46 | end 47 | 48 | letter = options.adapter[0].chr 49 | clazz_str = letter.upcase + options.adapter[1..-1].downcase 50 | clazz = Object.const_get( clazz_str + "Benchmark" ) 51 | 52 | benchmarks = [] 53 | options.number_of_objects.each do |num| 54 | benchmarks << (benchmark = clazz.new) 55 | benchmark.send( "benchmark", table_types, num ) 56 | end 57 | 58 | options.outputs.each do |output| 59 | format = output.format.downcase 60 | output_module = Object.const_get( "OutputTo#{format.upcase}" ) 61 | benchmarks.each do |benchmark| 62 | output_module.output_results( output.filename, benchmark.results ) 63 | end 64 | end 65 | 66 | puts 67 | puts "Done with benchmark!" 68 | -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/abstract_adapter.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord::Import::AbstractAdapter 2 | module InstanceMethods 3 | def next_value_for_sequence(sequence_name) 4 | %(#{sequence_name}.nextval) 5 | end 6 | 7 | def insert_many( sql, values, *args ) # :nodoc: 8 | number_of_inserts = 1 9 | 10 | base_sql, post_sql = if sql.is_a?( String ) 11 | [sql, ''] 12 | elsif sql.is_a?( Array ) 13 | [sql.shift, sql.join( ' ' )] 14 | end 15 | 16 | sql2insert = base_sql + values.join( ',' ) + post_sql 17 | insert( sql2insert, *args ) 18 | 19 | [number_of_inserts, []] 20 | end 21 | 22 | def pre_sql_statements(options) 23 | sql = [] 24 | sql << options[:pre_sql] if options[:pre_sql] 25 | sql << options[:command] if options[:command] 26 | sql << "IGNORE" if options[:ignore] 27 | 28 | # add keywords like IGNORE or DELAYED 29 | if options[:keywords].is_a?(Array) 30 | sql.concat(options[:keywords]) 31 | elsif options[:keywords] 32 | sql << options[:keywords].to_s 33 | end 34 | 35 | sql 36 | end 37 | 38 | # Synchronizes the passed in ActiveRecord instances with the records in 39 | # the database by calling +reload+ on each instance. 40 | def after_import_synchronize( instances ) 41 | instances.each(&:reload) 42 | end 43 | 44 | # Returns an array of post SQL statements given the passed in options. 45 | def post_sql_statements( table_name, options ) # :nodoc: 46 | post_sql_statements = [] 47 | 48 | if supports_on_duplicate_key_update? 49 | if options[:on_duplicate_key_ignore] && respond_to?(:sql_for_on_duplicate_key_ignore) 50 | post_sql_statements << sql_for_on_duplicate_key_ignore( table_name, options[:on_duplicate_key_ignore] ) 51 | elsif options[:on_duplicate_key_update] 52 | post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update] ) 53 | end 54 | end 55 | 56 | # custom user post_sql 57 | post_sql_statements << options[:post_sql] if options[:post_sql] 58 | 59 | # with rollup 60 | post_sql_statements << rollup_sql if options[:rollup] 61 | 62 | post_sql_statements 63 | end 64 | 65 | # Returns the maximum number of bytes that the server will allow 66 | # in a single packet 67 | def max_allowed_packet 68 | NO_MAX_PACKET 69 | end 70 | 71 | def supports_on_duplicate_key_update? 72 | false 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Ruby is copyrighted free software by Yukihiro Matsumoto . 2 | You can redistribute it and/or modify it under either the terms of the 3 | 2-clause BSDL (see the file BSDL), or the conditions below: 4 | 5 | 1. You may make and give away verbatim copies of the source form of the 6 | software without restriction, provided that you duplicate all of the 7 | original copyright notices and associated disclaimers. 8 | 9 | 2. You may modify your copy of the software in any way, provided that 10 | you do at least ONE of the following: 11 | 12 | a) place your modifications in the Public Domain or otherwise 13 | make them Freely Available, such as by posting said 14 | modifications to Usenet or an equivalent medium, or by allowing 15 | the author to include your modifications in the software. 16 | 17 | b) use the modified software only within your corporation or 18 | organization. 19 | 20 | c) give non-standard binaries non-standard names, with 21 | instructions on where to get the original software distribution. 22 | 23 | d) make other distribution arrangements with the author. 24 | 25 | 3. You may distribute the software in object code or binary form, 26 | provided that you do at least ONE of the following: 27 | 28 | a) distribute the binaries and library files of the software, 29 | together with instructions (in the manual page or equivalent) 30 | on where to get the original distribution. 31 | 32 | b) accompany the distribution with the machine-readable source of 33 | the software. 34 | 35 | c) give non-standard binaries non-standard names, with 36 | instructions on where to get the original software distribution. 37 | 38 | d) make other distribution arrangements with the author. 39 | 40 | 4. You may modify and include the part of the software into any other 41 | software (possibly commercial). But some files in the distribution 42 | are not written by the author, so that they are not under these terms. 43 | 44 | For the list of those files and their copying conditions, see the 45 | file LEGAL. 46 | 47 | 5. The scripts and library files supplied as input to or produced as 48 | output from the software do not automatically fall under the 49 | copyright of the software, but belong to whomever generated them, 50 | and may be sold commercially, and may be aggregated with this 51 | software. 52 | 53 | 6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR 54 | IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED 55 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 | PURPOSE. 57 | -------------------------------------------------------------------------------- /test/sqlite3/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | describe "#supports_imports?" do 4 | context "and SQLite is 3.7.11 or higher" do 5 | it "supports import" do 6 | version = ActiveRecord::ConnectionAdapters::SQLite3Adapter::Version.new("3.7.11") 7 | assert ActiveRecord::Base.supports_import?(version) 8 | 9 | version = ActiveRecord::ConnectionAdapters::SQLite3Adapter::Version.new("3.7.12") 10 | assert ActiveRecord::Base.supports_import?(version) 11 | end 12 | end 13 | 14 | context "and SQLite less than 3.7.11" do 15 | it "doesn't support import" do 16 | version = ActiveRecord::ConnectionAdapters::SQLite3Adapter::Version.new("3.7.10") 17 | assert !ActiveRecord::Base.supports_import?(version) 18 | end 19 | end 20 | end 21 | 22 | describe "#import" do 23 | it "imports with a single insert on SQLite 3.7.11 or higher" do 24 | assert_difference "Topic.count", +507 do 25 | result = Topic.import Build(7, :topics) 26 | assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed" 27 | assert_equal 7, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed" 28 | 29 | result = Topic.import Build(500, :topics) 30 | assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed" 31 | assert_equal 507, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed" 32 | end 33 | end 34 | 35 | it "imports with a two inserts on SQLite 3.7.11 or higher" do 36 | assert_difference "Topic.count", +501 do 37 | result = Topic.import Build(501, :topics) 38 | assert_equal 2, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed" 39 | assert_equal 501, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed" 40 | end 41 | end 42 | 43 | it "imports with a five inserts on SQLite 3.7.11 or higher" do 44 | assert_difference "Topic.count", +2500 do 45 | result = Topic.import Build(2500, :topics) 46 | assert_equal 5, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed" 47 | assert_equal 2500, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed" 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/activerecord-import/synchronize.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord # :nodoc: 2 | class Base # :nodoc: 3 | # Synchronizes the passed in ActiveRecord instances with data 4 | # from the database. This is like calling reload on an individual 5 | # ActiveRecord instance but it is intended for use on multiple instances. 6 | # 7 | # This uses one query for all instance updates and then updates existing 8 | # instances rather sending one query for each instance 9 | # 10 | # == Examples 11 | # # Synchronizing existing models by matching on the primary key field 12 | # posts = Post.where(author: "Zach").first 13 | # <.. out of system changes occur to change author name from Zach to Zachary..> 14 | # Post.synchronize posts 15 | # posts.first.author # => "Zachary" instead of Zach 16 | # 17 | # # Synchronizing using custom key fields 18 | # posts = Post.where(author: "Zach").first 19 | # <.. out of system changes occur to change the address of author 'Zach' to 1245 Foo Ln ..> 20 | # Post.synchronize posts, [:name] # queries on the :name column and not the :id column 21 | # posts.first.address # => "1245 Foo Ln" instead of whatever it was 22 | # 23 | def self.synchronize(instances, keys = [primary_key]) 24 | return if instances.empty? 25 | 26 | conditions = {} 27 | 28 | key_values = keys.map { |key| instances.map(&key.to_sym) } 29 | keys.zip(key_values).each { |key, values| conditions[key] = values } 30 | order = keys.map { |key| "#{key} ASC" }.join(",") 31 | 32 | klass = instances.first.class 33 | 34 | fresh_instances = klass.where(conditions).order(order) 35 | instances.each do |instance| 36 | matched_instance = fresh_instances.detect do |fresh_instance| 37 | keys.all? { |key| fresh_instance.send(key) == instance.send(key) } 38 | end 39 | 40 | next unless matched_instance 41 | 42 | instance.send :clear_aggregation_cache 43 | instance.send :clear_association_cache 44 | instance.instance_variable_set :@attributes, matched_instance.instance_variable_get(:@attributes) 45 | 46 | if instance.respond_to?(:clear_changes_information) 47 | instance.clear_changes_information # Rails 4.2 and higher 48 | else 49 | instance.instance_variable_set :@attributes_cache, {} # Rails 4.0, 4.1 50 | instance.changed_attributes.clear # Rails 3.1, 3.2 51 | instance.previous_changes.clear 52 | end 53 | 54 | # Since the instance now accurately reflects the record in 55 | # the database, ensure that instance.persisted? is true. 56 | instance.instance_variable_set '@new_record', false 57 | instance.instance_variable_set '@destroyed', false 58 | end 59 | end 60 | 61 | # See ActiveRecord::ConnectionAdapters::AbstractAdapter.synchronize 62 | def synchronize(instances, key = [ActiveRecord::Base.primary_key]) 63 | self.class.synchronize(instances, key) 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /test/support/assertions.rb: -------------------------------------------------------------------------------- 1 | class ActiveSupport::TestCase 2 | module ImportAssertions 3 | def self.extended(klass) 4 | klass.instance_eval do 5 | assertion(:should_not_update_created_at_on_timestamp_columns) do 6 | Timecop.freeze Chronic.parse("5 minutes from now") do 7 | perform_import 8 | assert_in_delta @topic.created_at.to_i, updated_topic.created_at.to_i, 1 9 | assert_in_delta @topic.created_on.to_i, updated_topic.created_on.to_i, 1 10 | end 11 | end 12 | 13 | assertion(:should_update_updated_at_on_timestamp_columns) do 14 | time = Chronic.parse("5 minutes from now") 15 | Timecop.freeze time do 16 | perform_import 17 | assert_in_delta time.to_i, updated_topic.updated_at.to_i, 1 18 | assert_in_delta time.to_i, updated_topic.updated_on.to_i, 1 19 | end 20 | end 21 | 22 | assertion(:should_not_update_updated_at_on_timestamp_columns) do 23 | time = Chronic.parse("5 minutes from now") 24 | Timecop.freeze time do 25 | perform_import 26 | assert_in_delta @topic.updated_at.to_i, updated_topic.updated_at.to_i, 1 27 | assert_in_delta @topic.updated_on.to_i, updated_topic.updated_on.to_i, 1 28 | end 29 | end 30 | 31 | assertion(:should_not_update_timestamps) do 32 | Timecop.freeze Chronic.parse("5 minutes from now") do 33 | perform_import timestamps: false 34 | assert_in_delta @topic.created_at.to_i, updated_topic.created_at.to_i, 1 35 | assert_in_delta @topic.created_on.to_i, updated_topic.created_on.to_i, 1 36 | assert_in_delta @topic.updated_at.to_i, updated_topic.updated_at.to_i, 1 37 | assert_in_delta @topic.updated_on.to_i, updated_topic.updated_on.to_i, 1 38 | end 39 | end 40 | 41 | assertion(:should_not_update_fields_not_mentioned) do 42 | assert_equal "John Doe", updated_topic.author_name 43 | end 44 | 45 | assertion(:should_update_fields_mentioned) do 46 | perform_import 47 | assert_equal "Book - 2nd Edition", updated_topic.title 48 | assert_equal "johndoe@example.com", updated_topic.author_email_address 49 | end 50 | 51 | assertion(:should_raise_update_fields_mentioned) do 52 | assert_raise ActiveRecord::RecordNotUnique do 53 | perform_import 54 | end 55 | 56 | assert_equal "Book", updated_topic.title 57 | assert_equal "john@doe.com", updated_topic.author_email_address 58 | end 59 | 60 | assertion(:should_update_fields_mentioned_with_hash_mappings) do 61 | perform_import 62 | assert_equal "johndoe@example.com", updated_topic.title 63 | assert_equal "Book - 2nd Edition", updated_topic.author_email_address 64 | end 65 | 66 | assertion(:should_update_foreign_keys) do 67 | perform_import 68 | assert_equal 57, updated_topic.parent_id 69 | end 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /benchmarks/lib/cli_parser.rb: -------------------------------------------------------------------------------- 1 | require 'optparse' 2 | require 'ostruct' 3 | 4 | # 5 | # == PARAMETERS 6 | # * a - database adapter. ie: mysql, postgresql, oracle, etc. 7 | # * n - number of objects to test with. ie: 1, 100, 1000, etc. 8 | # * t - the table types to test. ie: myisam, innodb, memory, temporary, etc. 9 | # 10 | module BenchmarkOptionParser 11 | BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options.".freeze 12 | 13 | def self.print_banner 14 | puts BANNER 15 | end 16 | 17 | def self.print_banner! 18 | print_banner 19 | exit 20 | end 21 | 22 | def self.print_options( options ) 23 | puts "Benchmarking the following options:" 24 | puts " Database adapter: #{options.adapter}" 25 | puts " Number of objects: #{options.number_of_objects}" 26 | puts " Table types:" 27 | print_valid_table_types( options, prefix: " " ) 28 | end 29 | 30 | # TODO IMPLEMENT THIS 31 | def self.print_valid_table_types( options, hsh = { prefix: '' } ) 32 | if !options.table_types.keys.empty? 33 | options.table_types.keys.sort.each { |type| puts hsh[:prefix].to_s + type.to_s } 34 | else 35 | puts 'No table types defined.' 36 | end 37 | end 38 | 39 | def self.parse( args ) 40 | options = OpenStruct.new( 41 | adapter: 'mysql2', 42 | table_types: {}, 43 | delete_on_finish: true, 44 | number_of_objects: [], 45 | outputs: [] ) 46 | 47 | opt_parser = OptionParser.new do |opts| 48 | opts.banner = BANNER 49 | 50 | # parse the database adapter 51 | opts.on( "a", "--adapter [String]", 52 | "The database adapter to use. IE: mysql, postgresql, oracle" ) do |arg| 53 | options.adapter = arg 54 | end 55 | 56 | # parse do_not_delete flag 57 | opts.on( "d", "--do-not-delete", 58 | "By default all records in the benchmark tables will be deleted at the end of the benchmark. " \ 59 | "This flag indicates not to delete the benchmark data." ) do |_| 60 | options.delete_on_finish = false 61 | end 62 | 63 | # parse the number of row objects to test 64 | opts.on( "n", "--num [Integer]", 65 | "The number of objects to benchmark." ) do |arg| 66 | options.number_of_objects << arg.to_i 67 | end 68 | 69 | # parse the table types to test 70 | opts.on( "t", "--table-type [String]", 71 | "The table type to test. This can be used multiple times." ) do |arg| 72 | if arg =~ /^all$/ 73 | options.table_types['all'] = options.benchmark_all_types = true 74 | else 75 | options.table_types[arg] = true 76 | end 77 | end 78 | 79 | # print results in CSV format 80 | opts.on( "--to-csv [String]", "Print results in a CSV file format" ) do |filename| 81 | options.outputs << OpenStruct.new( format: 'csv', filename: filename) 82 | end 83 | 84 | # print results in HTML format 85 | opts.on( "--to-html [String]", "Print results in HTML format" ) do |filename| 86 | options.outputs << OpenStruct.new( format: 'html', filename: filename ) 87 | end 88 | end # end opt.parse! 89 | 90 | begin 91 | opt_parser.parse!( args ) 92 | if options.table_types.empty? 93 | options.table_types['all'] = options.benchmark_all_types = true 94 | end 95 | rescue Exception 96 | print_banner! 97 | end 98 | 99 | options.number_of_objects = [1000] if options.number_of_objects.empty? 100 | options.outputs = [OpenStruct.new( format: 'html', filename: 'benchmark.html')] if options.outputs.empty? 101 | 102 | print_options( options ) 103 | 104 | options 105 | end 106 | end 107 | -------------------------------------------------------------------------------- /test/support/shared_examples/on_duplicate_key_update.rb: -------------------------------------------------------------------------------- 1 | def should_support_basic_on_duplicate_key_update 2 | describe "#import" do 3 | extend ActiveSupport::TestCase::ImportAssertions 4 | 5 | macro(:perform_import) { raise "supply your own #perform_import in a context below" } 6 | macro(:updated_topic) { Topic.find(@topic.id) } 7 | 8 | context "with :on_duplicate_key_update and validation checks turned off" do 9 | asssertion_group(:should_support_on_duplicate_key_update) do 10 | should_not_update_fields_not_mentioned 11 | should_update_foreign_keys 12 | should_not_update_created_at_on_timestamp_columns 13 | should_update_updated_at_on_timestamp_columns 14 | end 15 | 16 | let(:columns) { %w( id title author_name author_email_address parent_id ) } 17 | let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] } 18 | let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] } 19 | 20 | macro(:perform_import) do |*opts| 21 | Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: update_columns, validate: false) 22 | end 23 | 24 | setup do 25 | Topic.import columns, values, validate: false 26 | @topic = Topic.find 99 27 | end 28 | 29 | context "using an empty array" do 30 | let(:update_columns) { [] } 31 | should_not_update_fields_not_mentioned 32 | should_update_updated_at_on_timestamp_columns 33 | end 34 | 35 | context "using string column names" do 36 | let(:update_columns) { %w(title author_email_address parent_id) } 37 | should_support_on_duplicate_key_update 38 | should_update_fields_mentioned 39 | end 40 | 41 | context "using symbol column names" do 42 | let(:update_columns) { [:title, :author_email_address, :parent_id] } 43 | should_support_on_duplicate_key_update 44 | should_update_fields_mentioned 45 | end 46 | end 47 | 48 | context "with a table that has a non-standard primary key" do 49 | let(:columns) { [:promotion_id, :code] } 50 | let(:values) { [[1, 'DISCOUNT1']] } 51 | let(:updated_values) { [[1, 'DISCOUNT2']] } 52 | let(:update_columns) { [:code] } 53 | 54 | macro(:perform_import) do |*opts| 55 | Promotion.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: update_columns, validate: false) 56 | end 57 | macro(:updated_promotion) { Promotion.find(@promotion.promotion_id) } 58 | 59 | setup do 60 | Promotion.import columns, values, validate: false 61 | @promotion = Promotion.find 1 62 | end 63 | 64 | it "should update specified columns" do 65 | perform_import 66 | assert_equal 'DISCOUNT2', updated_promotion.code 67 | end 68 | end 69 | 70 | context "with :on_duplicate_key_update turned off" do 71 | let(:columns) { %w( id title author_name author_email_address parent_id ) } 72 | let(:values) { [[100, "Book", "John Doe", "john@doe.com", 17]] } 73 | let(:updated_values) { [[100, "Book - 2nd Edition", "This should raise an exception", "john@nogo.com", 57]] } 74 | 75 | macro(:perform_import) do |*opts| 76 | # `on_duplicate_key_update: false` is the tested feature 77 | Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: false, validate: false) 78 | end 79 | 80 | setup do 81 | Topic.import columns, values, validate: false 82 | @topic = Topic.find 100 83 | end 84 | 85 | it "should raise ActiveRecord::RecordNotUnique" do 86 | assert_raise ActiveRecord::RecordNotUnique do 87 | perform_import 88 | end 89 | end 90 | end 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /test/support/mysql/import_examples.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | def should_support_mysql_import_functionality 3 | # Forcefully disable strict mode for this session. 4 | ActiveRecord::Base.connection.execute "set sql_mode='STRICT_ALL_TABLES'" 5 | 6 | should_support_basic_on_duplicate_key_update 7 | 8 | describe "#import" do 9 | context "with :on_duplicate_key_update and validation checks turned off" do 10 | extend ActiveSupport::TestCase::ImportAssertions 11 | 12 | asssertion_group(:should_support_on_duplicate_key_update) do 13 | should_not_update_fields_not_mentioned 14 | should_update_foreign_keys 15 | should_not_update_created_at_on_timestamp_columns 16 | should_update_updated_at_on_timestamp_columns 17 | end 18 | 19 | macro(:perform_import) { raise "supply your own #perform_import in a context below" } 20 | macro(:updated_topic) { Topic.find(@topic.id) } 21 | 22 | let(:columns) { %w( id title author_name author_email_address parent_id ) } 23 | let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] } 24 | let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] } 25 | 26 | macro(:perform_import) do |*opts| 27 | Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: update_columns, validate: false) 28 | end 29 | 30 | setup do 31 | Topic.import columns, values, validate: false 32 | @topic = Topic.find 99 33 | end 34 | 35 | context "using string hash map" do 36 | let(:update_columns) { { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } } 37 | should_support_on_duplicate_key_update 38 | should_update_fields_mentioned 39 | end 40 | 41 | context "using string hash map, but specifying column mismatches" do 42 | let(:update_columns) { { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } } 43 | should_support_on_duplicate_key_update 44 | should_update_fields_mentioned_with_hash_mappings 45 | end 46 | 47 | context "using symbol hash map" do 48 | let(:update_columns) { { title: :title, author_email_address: :author_email_address, parent_id: :parent_id } } 49 | should_support_on_duplicate_key_update 50 | should_update_fields_mentioned 51 | end 52 | 53 | context "using symbol hash map, but specifying column mismatches" do 54 | let(:update_columns) { { title: :author_email_address, author_email_address: :title, parent_id: :parent_id } } 55 | should_support_on_duplicate_key_update 56 | should_update_fields_mentioned_with_hash_mappings 57 | end 58 | end 59 | 60 | context "with :synchronization option" do 61 | let(:topics) { [] } 62 | let(:values) { [[topics.first.id, "Jerry Carter", "title1"], [topics.last.id, "Chad Fowler", "title2"]] } 63 | let(:columns) { %w(id author_name title) } 64 | 65 | setup do 66 | topics << Topic.create!(title: "LDAP", author_name: "Big Bird") 67 | topics << Topic.create!(title: "Rails Recipes", author_name: "Elmo") 68 | end 69 | 70 | it "synchronizes passed in ActiveRecord model instances with the data just imported" do 71 | columns2update = ['author_name'] 72 | 73 | expected_count = Topic.count 74 | Topic.import( columns, values, 75 | validate: false, 76 | on_duplicate_key_update: columns2update, 77 | synchronize: topics ) 78 | 79 | assert_equal expected_count, Topic.count, "no new records should have been created!" 80 | assert_equal "Jerry Carter", topics.first.author_name, "wrong author!" 81 | assert_equal "Chad Fowler", topics.last.author_name, "wrong author!" 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /test/value_sets_bytes_parser_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/test_helper') 2 | 3 | require 'activerecord-import/value_sets_parser' 4 | 5 | describe ActiveRecord::Import::ValueSetsBytesParser do 6 | context "#parse - computing insert value sets" do 7 | let(:parser) { ActiveRecord::Import::ValueSetsBytesParser } 8 | let(:base_sql) { "INSERT INTO atable (a,b,c)" } 9 | let(:values) { ["(1,2,3)", "(2,3,4)", "(3,4,5)"] } 10 | 11 | context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do 12 | it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do 13 | value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 33 14 | assert_equal 3, value_sets.size 15 | end 16 | end 17 | 18 | context "when the max allowed bytes is 40 and the base SQL is 26 bytes" do 19 | it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do 20 | value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 40 21 | assert_equal 3, value_sets.size 22 | end 23 | end 24 | 25 | context "when the max allowed bytes is 41 and the base SQL is 26 bytes" do 26 | it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do 27 | value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 41 28 | assert_equal 2, value_sets.size 29 | end 30 | end 31 | 32 | context "when the max allowed bytes is 48 and the base SQL is 26 bytes" do 33 | it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do 34 | value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 48 35 | assert_equal 2, value_sets.size 36 | end 37 | end 38 | 39 | context "when the max allowed bytes is 49 and the base SQL is 26 bytes" do 40 | it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do 41 | value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 49 42 | assert_equal 1, value_sets.size 43 | end 44 | end 45 | 46 | context "when the max allowed bytes is 999999 and the base SQL is 26 bytes" do 47 | it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do 48 | value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 999_999 49 | assert_equal 1, value_sets.size 50 | end 51 | end 52 | 53 | it "should properly build insert value set based on max packet allowed" do 54 | values = [ 55 | "('1','2','3')", 56 | "('4','5','6')", 57 | "('7','8','9')"] 58 | 59 | base_sql_size_in_bytes = 15 60 | max_bytes = 30 61 | 62 | value_sets = parser.parse values, reserved_bytes: base_sql_size_in_bytes, max_bytes: max_bytes 63 | assert_equal 3, value_sets.size, 'Three value sets were expected!' 64 | 65 | # Each element in the value_sets array must be an array 66 | value_sets.each_with_index do |e, i| 67 | assert_kind_of Array, e, "Element #{i} was expected to be an Array!" 68 | end 69 | 70 | # Each element in the values array should have a 1:1 correlation to the elements 71 | # in the returned value_sets arrays 72 | assert_equal values[0], value_sets[0].first 73 | assert_equal values[1], value_sets[1].first 74 | assert_equal values[2], value_sets[2].first 75 | end 76 | 77 | context "data contains multi-byte chars" do 78 | it "should properly build insert value set based on max packet allowed" do 79 | # each accented e should be 2 bytes, so each entry is 6 bytes instead of 5 80 | values = [ 81 | "('é')", 82 | "('é')"] 83 | 84 | base_sql_size_in_bytes = 15 85 | max_bytes = 26 86 | 87 | value_sets = parser.parse values, reserved_bytes: base_sql_size_in_bytes, max_bytes: max_bytes 88 | 89 | assert_equal 2, value_sets.size, 'Two value sets were expected!' 90 | end 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /test/schema/generic_schema.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Schema.define do 2 | create_table :schema_info, force: :cascade do |t| 3 | t.integer :version, unique: true 4 | end 5 | SchemaInfo.create version: SchemaInfo::VERSION 6 | 7 | create_table :group, force: :cascade do |t| 8 | t.string :order 9 | t.timestamps null: true 10 | end 11 | 12 | create_table :topics, force: :cascade do |t| 13 | t.string :title, null: false 14 | t.string :author_name 15 | t.string :author_email_address 16 | t.datetime :written_on 17 | t.time :bonus_time 18 | t.datetime :last_read 19 | t.text :content 20 | t.boolean :approved, default: '1' 21 | t.integer :replies_count 22 | t.integer :parent_id 23 | t.string :type 24 | t.datetime :created_at 25 | t.datetime :created_on 26 | t.datetime :updated_at 27 | t.datetime :updated_on 28 | end 29 | 30 | create_table :projects, force: :cascade do |t| 31 | t.string :name 32 | t.string :type 33 | end 34 | 35 | create_table :developers, force: :cascade do |t| 36 | t.string :name 37 | t.integer :salary, default: '70000' 38 | t.datetime :created_at 39 | t.integer :team_id 40 | t.datetime :updated_at 41 | end 42 | 43 | create_table :addresses, force: :cascade do |t| 44 | t.string :address 45 | t.string :city 46 | t.string :state 47 | t.string :zip 48 | t.integer :developer_id 49 | end 50 | 51 | create_table :teams, force: :cascade do |t| 52 | t.string :name 53 | end 54 | 55 | create_table :books, force: :cascade do |t| 56 | t.string :title, null: false 57 | t.string :publisher, null: false, default: 'Default Publisher' 58 | t.string :author_name, null: false 59 | t.datetime :created_at 60 | t.datetime :created_on 61 | t.datetime :updated_at 62 | t.datetime :updated_on 63 | t.date :publish_date 64 | t.integer :topic_id 65 | t.boolean :for_sale, default: true 66 | t.integer :status, default: 0 67 | end 68 | 69 | create_table :chapters, force: :cascade do |t| 70 | t.string :title 71 | t.integer :book_id, null: false 72 | t.datetime :created_at 73 | t.datetime :updated_at 74 | end 75 | 76 | create_table :end_notes, force: :cascade do |t| 77 | t.string :note 78 | t.integer :book_id, null: false 79 | t.datetime :created_at 80 | t.datetime :updated_at 81 | end 82 | 83 | create_table :languages, force: :cascade do |t| 84 | t.string :name 85 | t.integer :developer_id 86 | end 87 | 88 | create_table :shopping_carts, force: :cascade do |t| 89 | t.string :name, null: true 90 | t.datetime :created_at 91 | t.datetime :updated_at 92 | end 93 | 94 | create_table :cart_items, force: :cascade do |t| 95 | t.string :shopping_cart_id, null: false 96 | t.string :book_id, null: false 97 | t.integer :copies, default: 1 98 | t.datetime :created_at 99 | t.datetime :updated_at 100 | end 101 | 102 | add_index :cart_items, [:shopping_cart_id, :book_id], unique: true, name: 'uk_shopping_cart_books' 103 | 104 | create_table :animals, force: :cascade do |t| 105 | t.string :name, null: false 106 | t.string :size, default: nil 107 | t.datetime :created_at 108 | t.datetime :updated_at 109 | end 110 | 111 | add_index :animals, [:name], unique: true, name: 'uk_animals' 112 | 113 | create_table :widgets, id: false, force: :cascade do |t| 114 | t.integer :w_id 115 | t.boolean :active, default: false 116 | t.text :data 117 | t.text :json_data 118 | end 119 | 120 | create_table :promotions, primary_key: :promotion_id, force: :cascade do |t| 121 | t.string :code 122 | t.string :description 123 | t.decimal :discount 124 | end 125 | 126 | add_index :promotions, [:code], unique: true, name: 'uk_code' 127 | 128 | create_table :discounts, force: :cascade do |t| 129 | t.decimal :amount 130 | t.integer :discountable_id 131 | t.string :discountable_type 132 | end 133 | 134 | create_table :rules, force: :cascade do |t| 135 | t.string :condition_text 136 | t.integer :question_id 137 | end 138 | 139 | create_table :questions, force: :cascade do |t| 140 | t.string :body 141 | end 142 | end 143 | -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/mysql_adapter.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord::Import::MysqlAdapter 2 | include ActiveRecord::Import::ImportSupport 3 | include ActiveRecord::Import::OnDuplicateKeyUpdateSupport 4 | 5 | NO_MAX_PACKET = 0 6 | QUERY_OVERHEAD = 8 # This was shown to be true for MySQL, but it's not clear where the overhead is from. 7 | 8 | # +sql+ can be a single string or an array. If it is an array all 9 | # elements that are in position >= 1 will be appended to the final SQL. 10 | def insert_many( sql, values, *args ) # :nodoc: 11 | # the number of inserts default 12 | number_of_inserts = 0 13 | 14 | base_sql, post_sql = if sql.is_a?( String ) 15 | [sql, ''] 16 | elsif sql.is_a?( Array ) 17 | [sql.shift, sql.join( ' ' )] 18 | end 19 | 20 | sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size 21 | 22 | # the number of bytes the requested insert statement values will take up 23 | values_in_bytes = values.sum(&:bytesize) 24 | 25 | # the number of bytes (commas) it will take to comma separate our values 26 | comma_separated_bytes = values.size - 1 27 | 28 | # the total number of bytes required if this statement is one statement 29 | total_bytes = sql_size + values_in_bytes + comma_separated_bytes 30 | 31 | max = max_allowed_packet 32 | 33 | # if we can insert it all as one statement 34 | if NO_MAX_PACKET == max || total_bytes < max 35 | number_of_inserts += 1 36 | sql2insert = base_sql + values.join( ',' ) + post_sql 37 | insert( sql2insert, *args ) 38 | else 39 | value_sets = ::ActiveRecord::Import::ValueSetsBytesParser.parse(values, 40 | reserved_bytes: sql_size, 41 | max_bytes: max) 42 | value_sets.each do |value_set| 43 | number_of_inserts += 1 44 | sql2insert = base_sql + value_set.join( ',' ) + post_sql 45 | insert( sql2insert, *args ) 46 | end 47 | end 48 | 49 | [number_of_inserts, []] 50 | end 51 | 52 | # Returns the maximum number of bytes that the server will allow 53 | # in a single packet 54 | def max_allowed_packet # :nodoc: 55 | @max_allowed_packet ||= begin 56 | result = execute( "SHOW VARIABLES like 'max_allowed_packet';" ) 57 | # original Mysql gem responds to #fetch_row while Mysql2 responds to #first 58 | val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1] 59 | val.to_i 60 | end 61 | end 62 | 63 | # Add a column to be updated on duplicate key update 64 | def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc: 65 | if options.include?(:on_duplicate_key_update) 66 | columns = options[:on_duplicate_key_update] 67 | case columns 68 | when Array then columns << column.to_sym unless columns.include?(column.to_sym) 69 | when Hash then columns[column.to_sym] = column.to_sym 70 | end 71 | else 72 | options[:on_duplicate_key_update] = [column.to_sym] 73 | end 74 | end 75 | 76 | # Returns a generated ON DUPLICATE KEY UPDATE statement given the passed 77 | # in +args+. 78 | def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc: 79 | sql = ' ON DUPLICATE KEY UPDATE ' 80 | arg = args.first 81 | if arg.is_a?( Array ) 82 | sql << sql_for_on_duplicate_key_update_as_array( table_name, arg ) 83 | elsif arg.is_a?( Hash ) 84 | sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg ) 85 | elsif arg.is_a?( String ) 86 | sql << arg 87 | else 88 | raise ArgumentError, "Expected Array or Hash" 89 | end 90 | sql 91 | end 92 | 93 | def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc: 94 | results = arr.map do |column| 95 | qc = quote_column_name( column ) 96 | "#{table_name}.#{qc}=VALUES(#{qc})" 97 | end 98 | results.join( ',' ) 99 | end 100 | 101 | def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc: 102 | results = hsh.map do |column1, column2| 103 | qc1 = quote_column_name( column1 ) 104 | qc2 = quote_column_name( column2 ) 105 | "#{table_name}.#{qc1}=VALUES( #{qc2} )" 106 | end 107 | results.join( ',') 108 | end 109 | 110 | # Return true if the statement is a duplicate key record error 111 | def duplicate_key_update_error?(exception) # :nodoc: 112 | exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry') 113 | end 114 | end 115 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # activerecord-import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import) 2 | 3 | activerecord-import is a library for bulk inserting data using ActiveRecord. 4 | 5 | One of its major features is following activerecord associations and generating the minimal 6 | number of SQL insert statements required, avoiding the N+1 insert problem. An example probably 7 | explains it best. Say you had a schema like this: 8 | 9 | - Publishers have Books 10 | - Books have Reviews 11 | 12 | and you wanted to bulk insert 100 new publishers with 10K books and 3 reviews per book. This library will follow the associations 13 | down and generate only 3 SQL insert statements - one for the publishers, one for the books, and one for the reviews. 14 | 15 | In contrast, the standard ActiveRecord save would generate 16 | 100 insert statements for the publishers, then it would visit each publisher and save all the books: 17 | 100 * 10,000 = 1,000,000 SQL insert statements 18 | and then the reviews: 19 | 100 * 10,000 * 3 = 3M SQL insert statements, 20 | 21 | That would be about 4M SQL insert statements vs 3, which results in vastly improved performance. In our case, it converted 22 | an 18 hour batch process to <2 hrs. 23 | 24 | ### Rails 5.0 25 | 26 | Use activerecord-import 0.11.0 or higher. 27 | 28 | ### Rails 4.0 29 | 30 | Use activerecord-import 0.4.0 or higher. 31 | 32 | ### Rails 3.1.x up to, but not including 4.0 33 | 34 | Use the latest in the activerecord-import 0.3.x series. 35 | 36 | ### Rails 3.0.x up to, but not including 3.1 37 | 38 | Use activerecord-import 0.2.11. As of activerecord-import 0.3.0 we are relying on functionality that was introduced in Rails 3.1. Since Rails 3.0.x is no longer a supported version of Rails we have decided to drop support as well. 39 | 40 | ### For More Information 41 | 42 | For more information on activerecord-import please see its wiki: https://github.com/zdennis/activerecord-import/wiki 43 | 44 | ## Additional Adapters 45 | Additional adapters can be provided by gems external to activerecord-import by providing an adapter that matches the naming convention setup by activerecord-import (and subsequently activerecord) for dynamically loading adapters. This involves also providing a folder on the load path that follows the activerecord-import naming convention to allow activerecord-import to dynamically load the file. 46 | 47 | When `ActiveRecord::Import.require_adapter("fake_name")` is called the require will be: 48 | 49 | ```ruby 50 | require 'activerecord-import/active_record/adapters/fake_name_adapter' 51 | ``` 52 | 53 | This allows an external gem to dyanmically add an adapter without the need to add any file/code to the core activerecord-import gem. 54 | 55 | ### Load Path Setup 56 | To understand how rubygems loads code you can reference the following: 57 | 58 | http://guides.rubygems.org/patterns/#loading_code 59 | 60 | And an example of how active_record dynamically load adapters: 61 | https://github.com/rails/rails/blob/master/activerecord/lib/active_record/connection_adapters/connection_specification.rb 62 | 63 | In summary, when a gem is loaded rubygems adds the `lib` folder of the gem to the global load path `$LOAD_PATH` so that all `require` lookups will not propegate through all of the folders on the load path. When a `require` is issued each folder on the `$LOAD_PATH` is checked for the file and/or folder referenced. This allows a gem (like activerecord-import) to define push the activerecord-import folder (or namespace) on the `$LOAD_PATH` and any adapters provided by activerecord-import will be found by rubygems when the require is issued. 64 | 65 | If `fake_name` adapter is needed by a gem (potentially called `activerecord-import-fake_name`) then the folder structure should look as follows: 66 | 67 | ```bash 68 | activerecord-import-fake_name/ 69 | |-- activerecord-import-fake_name.gemspec 70 | |-- lib 71 | | |-- activerecord-import-fake_name 72 | | | |-- version.rb 73 | | |-- activerecord-import 74 | | | |-- active_record 75 | | | | |-- adapters 76 | | | | |-- fake_name_adapter.rb 77 | |--activerecord-import-fake_name.rb 78 | ``` 79 | 80 | When rubygems pushes the `lib` folder onto the load path a `require` will now find `activerecord-import/active_record/adapters/fake_name_adapter` as it runs through the lookup process for a ruby file under that path in `$LOAD_PATH` 81 | 82 | # License 83 | 84 | This is licensed under the ruby license. 85 | 86 | # Author 87 | 88 | Zach Dennis (zach.dennis@gmail.com) 89 | 90 | # Contributors 91 | 92 | * Jordan Owens (@jkowens) 93 | * Erik Michaels-Ober (@sferik) 94 | * Blythe Dunham 95 | * Gabe da Silveira 96 | * Henry Work 97 | * James Herdman 98 | * Marcus Crafter 99 | * Thibaud Guillaume-Gentil 100 | * Mark Van Holstyn 101 | * Victor Costan 102 | -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/postgresql_adapter.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord::Import::PostgreSQLAdapter 2 | include ActiveRecord::Import::ImportSupport 3 | include ActiveRecord::Import::OnDuplicateKeyUpdateSupport 4 | 5 | MIN_VERSION_FOR_UPSERT = 90_500 6 | 7 | def insert_many( sql, values, *args ) # :nodoc: 8 | number_of_inserts = 1 9 | 10 | base_sql, post_sql = if sql.is_a?( String ) 11 | [sql, ''] 12 | elsif sql.is_a?( Array ) 13 | [sql.shift, sql.join( ' ' )] 14 | end 15 | 16 | sql2insert = base_sql + values.join( ',' ) + post_sql 17 | ids = select_values( sql2insert, *args ) 18 | 19 | ActiveRecord::Base.connection.query_cache.clear 20 | 21 | [number_of_inserts, ids] 22 | end 23 | 24 | def next_value_for_sequence(sequence_name) 25 | %{nextval('#{sequence_name}')} 26 | end 27 | 28 | def post_sql_statements( table_name, options ) # :nodoc: 29 | if options[:primary_key].blank? 30 | super(table_name, options) 31 | else 32 | super(table_name, options) << "RETURNING #{options[:primary_key]}" 33 | end 34 | end 35 | 36 | # Add a column to be updated on duplicate key update 37 | def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc: 38 | arg = options[:on_duplicate_key_update] 39 | if arg.is_a?( Hash ) 40 | columns = arg.fetch( :columns ) { arg[:columns] = [] } 41 | case columns 42 | when Array then columns << column.to_sym unless columns.include?( column.to_sym ) 43 | when Hash then columns[column.to_sym] = column.to_sym 44 | end 45 | elsif arg.is_a?( Array ) 46 | arg << column.to_sym unless arg.include?( column.to_sym ) 47 | end 48 | end 49 | 50 | # Returns a generated ON CONFLICT DO NOTHING statement given the passed 51 | # in +args+. 52 | def sql_for_on_duplicate_key_ignore( table_name, *args ) # :nodoc: 53 | arg = args.first 54 | conflict_target = sql_for_conflict_target( arg ) if arg.is_a?( Hash ) 55 | " ON CONFLICT #{conflict_target}DO NOTHING" 56 | end 57 | 58 | # Returns a generated ON CONFLICT DO UPDATE statement given the passed 59 | # in +args+. 60 | def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc: 61 | arg = args.first 62 | arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String ) 63 | return unless arg.is_a?( Hash ) 64 | 65 | sql = " ON CONFLICT " 66 | conflict_target = sql_for_conflict_target( arg ) 67 | 68 | columns = arg.fetch( :columns, [] ) 69 | if columns.respond_to?( :empty? ) && columns.empty? 70 | return sql << "#{conflict_target}DO NOTHING" 71 | end 72 | 73 | conflict_target ||= sql_for_default_conflict_target( table_name ) 74 | unless conflict_target 75 | raise ArgumentError, 'Expected :conflict_target or :constraint_name to be specified' 76 | end 77 | 78 | sql << "#{conflict_target}DO UPDATE SET " 79 | if columns.is_a?( Array ) 80 | sql << sql_for_on_duplicate_key_update_as_array( table_name, columns ) 81 | elsif columns.is_a?( Hash ) 82 | sql << sql_for_on_duplicate_key_update_as_hash( table_name, columns ) 83 | elsif columns.is_a?( String ) 84 | sql << columns 85 | else 86 | raise ArgumentError, 'Expected :columns to be an Array or Hash' 87 | end 88 | sql 89 | end 90 | 91 | def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc: 92 | results = arr.map do |column| 93 | qc = quote_column_name( column ) 94 | "#{qc}=EXCLUDED.#{qc}" 95 | end 96 | results.join( ',' ) 97 | end 98 | 99 | def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc: 100 | results = hsh.map do |column1, column2| 101 | qc1 = quote_column_name( column1 ) 102 | qc2 = quote_column_name( column2 ) 103 | "#{qc1}=EXCLUDED.#{qc2}" 104 | end 105 | results.join( ',' ) 106 | end 107 | 108 | def sql_for_conflict_target( args = {} ) 109 | constraint_name = args[:constraint_name] 110 | conflict_target = args[:conflict_target] 111 | if constraint_name 112 | "ON CONSTRAINT #{constraint_name} " 113 | elsif conflict_target 114 | '(' << Array( conflict_target ).join( ', ' ) << ') ' 115 | end 116 | end 117 | 118 | def sql_for_default_conflict_target( table_name ) 119 | "(#{primary_key( table_name )}) " 120 | end 121 | 122 | # Return true if the statement is a duplicate key record error 123 | def duplicate_key_update_error?(exception) # :nodoc: 124 | exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key') 125 | end 126 | 127 | def supports_on_duplicate_key_update?(current_version = postgresql_version) 128 | current_version >= MIN_VERSION_FOR_UPSERT 129 | end 130 | 131 | def supports_on_duplicate_key_ignore?(current_version = postgresql_version) 132 | supports_on_duplicate_key_update?(current_version) 133 | end 134 | 135 | def support_setting_primary_key_of_imported_objects? 136 | true 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /benchmarks/lib/base.rb: -------------------------------------------------------------------------------- 1 | class BenchmarkBase 2 | attr_reader :results 3 | 4 | # The main benchmark method dispatcher. This dispatches the benchmarks 5 | # to actual benchmark_xxxx methods. 6 | # 7 | # == PARAMETERS 8 | # * table_types - an array of table types to benchmark 9 | # * num - the number of record insertions to test 10 | def benchmark( table_types, num ) 11 | array_of_cols_and_vals = build_array_of_cols_and_vals( num ) 12 | table_types.each do |table_type| 13 | send( "benchmark_#{table_type}", array_of_cols_and_vals ) 14 | end 15 | end 16 | 17 | # Returns an OpenStruct which contains two attritues, +description+ and +tms+ after performing an 18 | # actual benchmark. 19 | # 20 | # == PARAMETERS 21 | # * description - the description of the block that is getting benchmarked 22 | # * blk - the block of code to benchmark 23 | # 24 | # == RETURNS 25 | # An OpenStruct object with the following attributes: 26 | # * description - the description of the benchmark ran 27 | # * tms - a Benchmark::Tms containing the results of the benchmark 28 | def bm( description ) 29 | tms = nil 30 | puts "Benchmarking #{description}" 31 | 32 | Benchmark.bm { |x| tms = x.report { yield } } 33 | delete_all 34 | failed = false 35 | 36 | OpenStruct.new description: description, tms: tms, failed: failed 37 | end 38 | 39 | # Given a model class (ie: Topic), and an array of columns and value sets 40 | # this will perform all of the benchmarks necessary for this library. 41 | # 42 | # == PARAMETERS 43 | # * model_clazz - the model class to benchmark (ie: Topic) 44 | # * array_of_cols_and_vals - an array of column identifiers and value sets 45 | # 46 | # == RETURNS 47 | # returns true 48 | def bm_model( model_clazz, array_of_cols_and_vals ) 49 | puts 50 | puts "------ Benchmarking #{model_clazz.name} -------" 51 | 52 | cols, vals = array_of_cols_and_vals 53 | num_inserts = vals.size 54 | 55 | # add a new result group for this particular benchmark 56 | group = [] 57 | @results << group 58 | 59 | description = "#{model_clazz.name}.create (#{num_inserts} records)" 60 | group << bm( description ) do 61 | vals.each do |values| 62 | model_clazz.create create_hash_for_cols_and_vals( cols, values ) 63 | end 64 | end 65 | 66 | description = "#{model_clazz.name}.import(column, values) for #{num_inserts} records with validations" 67 | group << bm( description ) { model_clazz.import cols, vals, validate: true } 68 | 69 | description = "#{model_clazz.name}.import(columns, values) for #{num_inserts} records without validations" 70 | group << bm( description ) { model_clazz.import cols, vals, validate: false } 71 | 72 | models = [] 73 | array_of_attrs = [] 74 | 75 | vals.each do |arr| 76 | array_of_attrs << (attrs = {}) 77 | arr.each_with_index { |value, i| attrs[cols[i]] = value } 78 | end 79 | array_of_attrs.each { |attrs| models << model_clazz.new(attrs) } 80 | 81 | description = "#{model_clazz.name}.import(models) for #{num_inserts} records with validations" 82 | group << bm( description ) { model_clazz.import models, validate: true } 83 | 84 | description = "#{model_clazz.name}.import(models) for #{num_inserts} records without validations" 85 | group << bm( description ) { model_clazz.import models, validate: false } 86 | 87 | true 88 | end 89 | 90 | # Returns a two element array composing of an array of columns and an array of 91 | # value sets given the passed +num+. 92 | # 93 | # === What is a value set? 94 | # A value set is an array of arrays. Each child array represents an array of value sets 95 | # for a given row of data. 96 | # 97 | # For example, say we wanted to represent an insertion of two records: 98 | # column_names = [ 'id', 'name', 'description' ] 99 | # record1 = [ 1, 'John Doe', 'A plumber' ] 100 | # record2 = [ 2, 'John Smith', 'A painter' ] 101 | # value_set [ record1, record2 ] 102 | # 103 | # == PARAMETER 104 | # * num - the number of records to create 105 | def build_array_of_cols_and_vals( num ) 106 | cols = [:my_name, :description] 107 | value_sets = [] 108 | num.times { |i| value_sets << ["My Name #{i}", "My Description #{i}"] } 109 | [cols, value_sets] 110 | end 111 | 112 | # Returns a hash of column identifier to value mappings giving the passed in 113 | # value array. 114 | # 115 | # Example: 116 | # cols = [ 'id', 'name', 'description' ] 117 | # values = [ 1, 'John Doe', 'A plumber' ] 118 | # hsh = create_hash_for_cols_and_vals( cols, values ) 119 | # # hsh => { 'id'=>1, 'name'=>'John Doe', 'description'=>'A plumber' } 120 | def create_hash_for_cols_and_vals( cols, vals ) 121 | h = {} 122 | cols.zip( vals ) { |col, val| h[col] = val } 123 | h 124 | end 125 | 126 | # Deletes all records from all ActiveRecord subclasses 127 | def delete_all 128 | ActiveRecord::Base.send( :subclasses ).each do |subclass| 129 | subclass.delete_all if subclass.respond_to? :delete_all 130 | end 131 | end 132 | 133 | def initialize # :nodoc: 134 | @results = [] 135 | end 136 | end 137 | -------------------------------------------------------------------------------- /test/support/postgresql/import_examples.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | def should_support_postgresql_import_functionality 3 | describe "#supports_imports?" do 4 | it "should support import" do 5 | assert ActiveRecord::Base.supports_import? 6 | end 7 | end 8 | 9 | describe "#import" do 10 | it "should import with a single insert" do 11 | # see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics 12 | assert_difference "Topic.count", +10 do 13 | result = Topic.import Build(3, :topics) 14 | assert_equal 1, result.num_inserts 15 | 16 | result = Topic.import Build(7, :topics) 17 | assert_equal 1, result.num_inserts 18 | end 19 | end 20 | 21 | describe "importing objects with associations" do 22 | let(:new_topics) { Build(num_topics, :topic_with_book) } 23 | let(:new_topics_with_invalid_chapter) do 24 | chapter = new_topics.first.books.first.chapters.first 25 | chapter.title = nil 26 | new_topics 27 | end 28 | let(:num_topics) { 3 } 29 | let(:num_books) { 6 } 30 | let(:num_chapters) { 18 } 31 | let(:num_endnotes) { 24 } 32 | 33 | let(:new_question_with_rule) { FactoryGirl.build :question, :with_rule } 34 | 35 | it 'imports top level' do 36 | assert_difference "Topic.count", +num_topics do 37 | Topic.import new_topics, recursive: true 38 | new_topics.each do |topic| 39 | assert_not_nil topic.id 40 | end 41 | end 42 | end 43 | 44 | it 'imports first level associations' do 45 | assert_difference "Book.count", +num_books do 46 | Topic.import new_topics, recursive: true 47 | new_topics.each do |topic| 48 | topic.books.each do |book| 49 | assert_equal topic.id, book.topic_id 50 | end 51 | end 52 | end 53 | end 54 | 55 | it 'imports polymorphic associations' do 56 | discounts = Array.new(1) { |i| Discount.new(amount: i) } 57 | books = Array.new(1) { |i| Book.new(author_name: "Author ##{i}", title: "Book ##{i}") } 58 | books.each do |book| 59 | book.discounts << discounts 60 | end 61 | Book.import books, recursive: true 62 | books.each do |book| 63 | book.discounts.each do |discount| 64 | assert_not_nil discount.discountable_id 65 | assert_equal 'Book', discount.discountable_type 66 | end 67 | end 68 | end 69 | 70 | [{ recursive: false }, {}].each do |import_options| 71 | it "skips recursion for #{import_options}" do 72 | assert_difference "Book.count", 0 do 73 | Topic.import new_topics, import_options 74 | end 75 | end 76 | end 77 | 78 | it 'imports deeper nested associations' do 79 | assert_difference "Chapter.count", +num_chapters do 80 | assert_difference "EndNote.count", +num_endnotes do 81 | Topic.import new_topics, recursive: true 82 | new_topics.each do |topic| 83 | topic.books.each do |book| 84 | book.chapters.each do |chapter| 85 | assert_equal book.id, chapter.book_id 86 | end 87 | book.end_notes.each do |endnote| 88 | assert_equal book.id, endnote.book_id 89 | end 90 | end 91 | end 92 | end 93 | end 94 | end 95 | 96 | it "skips validation of the associations if requested" do 97 | assert_difference "Chapter.count", +num_chapters do 98 | Topic.import new_topics_with_invalid_chapter, validate: false, recursive: true 99 | end 100 | end 101 | 102 | it 'imports has_one associations' do 103 | assert_difference 'Rule.count' do 104 | Question.import [new_question_with_rule], recursive: true 105 | end 106 | end 107 | 108 | # These models dont validate associated. So we expect that books and topics get inserted, but not chapters 109 | # Putting a transaction around everything wouldn't work, so if you want your chapters to prevent topics from 110 | # being created, you would need to have validates_associated in your models and insert with validation 111 | describe "all_or_none" do 112 | [Book, Topic, EndNote].each do |type| 113 | it "creates #{type}" do 114 | assert_difference "#{type}.count", send("num_#{type.to_s.downcase}s") do 115 | Topic.import new_topics_with_invalid_chapter, all_or_none: true, recursive: true 116 | end 117 | end 118 | end 119 | it "doesn't create chapters" do 120 | assert_difference "Chapter.count", 0 do 121 | Topic.import new_topics_with_invalid_chapter, all_or_none: true, recursive: true 122 | end 123 | end 124 | end 125 | end 126 | 127 | describe "with query cache enabled" do 128 | setup do 129 | unless ActiveRecord::Base.connection.query_cache_enabled 130 | ActiveRecord::Base.connection.enable_query_cache! 131 | @disable_cache_on_teardown = true 132 | end 133 | end 134 | 135 | it "clears cache on insert" do 136 | before_import = Topic.all.to_a 137 | 138 | Topic.import(Build(2, :topics), validate: false) 139 | 140 | after_import = Topic.all.to_a 141 | assert_equal 2, after_import.size - before_import.size 142 | end 143 | 144 | teardown do 145 | if @disable_cache_on_teardown 146 | ActiveRecord::Base.connection.disable_query_cache! 147 | end 148 | end 149 | end 150 | end 151 | end 152 | 153 | def should_support_postgresql_upsert_functionality 154 | should_support_basic_on_duplicate_key_update 155 | 156 | describe "#import" do 157 | extend ActiveSupport::TestCase::ImportAssertions 158 | 159 | macro(:perform_import) { raise "supply your own #perform_import in a context below" } 160 | macro(:updated_topic) { Topic.find(@topic.id) } 161 | 162 | context "with :on_duplicate_key_ignore and validation checks turned off" do 163 | let(:columns) { %w( id title author_name author_email_address parent_id ) } 164 | let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] } 165 | let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] } 166 | 167 | macro(:perform_import) do |*opts| 168 | Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_ignore: value, validate: false) 169 | end 170 | 171 | setup do 172 | Topic.import columns, values, validate: false 173 | @topic = Topic.find 99 174 | end 175 | 176 | context "using true" do 177 | let(:value) { true } 178 | should_not_update_updated_at_on_timestamp_columns 179 | end 180 | 181 | context "using hash with :conflict_target" do 182 | let(:value) { { conflict_target: :id } } 183 | should_not_update_updated_at_on_timestamp_columns 184 | end 185 | 186 | context "using hash with :constraint_target" do 187 | let(:value) { { constraint_name: :topics_pkey } } 188 | should_not_update_updated_at_on_timestamp_columns 189 | end 190 | end 191 | 192 | context "with :on_duplicate_key_update and validation checks turned off" do 193 | asssertion_group(:should_support_on_duplicate_key_update) do 194 | should_not_update_fields_not_mentioned 195 | should_update_foreign_keys 196 | should_not_update_created_at_on_timestamp_columns 197 | should_update_updated_at_on_timestamp_columns 198 | end 199 | 200 | context "using a hash" do 201 | context "with :columns a hash" do 202 | let(:columns) { %w( id title author_name author_email_address parent_id ) } 203 | let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] } 204 | let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] } 205 | 206 | macro(:perform_import) do |*opts| 207 | Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id, columns: update_columns }, validate: false) 208 | end 209 | 210 | setup do 211 | Topic.import columns, values, validate: false 212 | @topic = Topic.find 99 213 | end 214 | 215 | context "using string hash map" do 216 | let(:update_columns) { { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } } 217 | should_support_on_duplicate_key_update 218 | should_update_fields_mentioned 219 | end 220 | 221 | context "using string hash map, but specifying column mismatches" do 222 | let(:update_columns) { { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } } 223 | should_support_on_duplicate_key_update 224 | should_update_fields_mentioned_with_hash_mappings 225 | end 226 | 227 | context "using symbol hash map" do 228 | let(:update_columns) { { title: :title, author_email_address: :author_email_address, parent_id: :parent_id } } 229 | should_support_on_duplicate_key_update 230 | should_update_fields_mentioned 231 | end 232 | 233 | context "using symbol hash map, but specifying column mismatches" do 234 | let(:update_columns) { { title: :author_email_address, author_email_address: :title, parent_id: :parent_id } } 235 | should_support_on_duplicate_key_update 236 | should_update_fields_mentioned_with_hash_mappings 237 | end 238 | end 239 | 240 | context "with :constraint_name" do 241 | let(:columns) { %w( id title author_name author_email_address parent_id ) } 242 | let(:values) { [[100, "Book", "John Doe", "john@doe.com", 17]] } 243 | let(:updated_values) { [[100, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] } 244 | 245 | macro(:perform_import) do |*opts| 246 | Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { constraint_name: :topics_pkey, columns: update_columns }, validate: false) 247 | end 248 | 249 | setup do 250 | Topic.import columns, values, validate: false 251 | @topic = Topic.find 100 252 | end 253 | 254 | let(:update_columns) { [:title, :author_email_address, :parent_id] } 255 | should_support_on_duplicate_key_update 256 | should_update_fields_mentioned 257 | end 258 | 259 | context "with no :conflict_target or :constraint_name" do 260 | let(:columns) { %w( id title author_name author_email_address parent_id ) } 261 | let(:values) { [[100, "Book", "John Doe", "john@doe.com", 17]] } 262 | let(:updated_values) { [[100, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] } 263 | 264 | macro(:perform_import) do |*opts| 265 | Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { columns: update_columns }, validate: false) 266 | end 267 | 268 | setup do 269 | Topic.import columns, values, validate: false 270 | @topic = Topic.find 100 271 | end 272 | 273 | context "default to the primary key" do 274 | let(:update_columns) { [:title, :author_email_address, :parent_id] } 275 | should_support_on_duplicate_key_update 276 | should_update_fields_mentioned 277 | end 278 | end 279 | 280 | context "with no :columns" do 281 | let(:columns) { %w( id title author_name author_email_address ) } 282 | let(:values) { [[100, "Book", "John Doe", "john@doe.com"]] } 283 | let(:updated_values) { [[100, "Title Should Not Change", "Author Should Not Change", "john@nogo.com"]] } 284 | 285 | macro(:perform_import) do |*opts| 286 | Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id }, validate: false) 287 | end 288 | 289 | setup do 290 | Topic.import columns, values, validate: false 291 | @topic = Topic.find 100 292 | end 293 | 294 | should_update_updated_at_on_timestamp_columns 295 | end 296 | end 297 | 298 | context "with recursive: true" do 299 | let(:new_topics) { Build(1, :topic_with_book) } 300 | 301 | it "imports objects with associations" do 302 | assert_difference "Topic.count", +1 do 303 | Topic.import new_topics, recursive: true, on_duplicate_key_update: [:updated_at], validate: false 304 | new_topics.each do |topic| 305 | assert_not_nil topic.id 306 | end 307 | end 308 | end 309 | end 310 | end 311 | end 312 | end 313 | -------------------------------------------------------------------------------- /test/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../test_helper', __FILE__) 2 | 3 | describe "#import" do 4 | it "should return the number of inserts performed" do 5 | # see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics 6 | assert_difference "Topic.count", +10 do 7 | result = Topic.import Build(3, :topics) 8 | assert result.num_inserts > 0 9 | 10 | result = Topic.import Build(7, :topics) 11 | assert result.num_inserts > 0 12 | end 13 | end 14 | 15 | it "should not produce an error when importing empty arrays" do 16 | assert_nothing_raised do 17 | Topic.import [] 18 | Topic.import %w(title author_name), [] 19 | end 20 | end 21 | 22 | describe "argument safety" do 23 | it "should not modify the passed in columns array" do 24 | assert_nothing_raised do 25 | columns = %w(title author_name).freeze 26 | Topic.import columns, [%w(foo bar)] 27 | end 28 | end 29 | 30 | it "should not modify the passed in values array" do 31 | assert_nothing_raised do 32 | values = [%w(foo bar)].freeze 33 | Topic.import %w(title author_name), values 34 | end 35 | end 36 | end 37 | 38 | describe "with non-default ActiveRecord models" do 39 | context "that have a non-standard primary key (that is no sequence)" do 40 | it "should import models successfully" do 41 | assert_difference "Widget.count", +3 do 42 | Widget.import Build(3, :widgets) 43 | end 44 | end 45 | end 46 | end 47 | 48 | context "with :validation option" do 49 | let(:columns) { %w(title author_name) } 50 | let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] } 51 | let(:valid_values_with_context) { [[1111, "Jerry Carter"], [2222, "Chad Fowler"]] } 52 | let(:invalid_values) { [["The RSpec Book", ""], ["Agile+UX", ""]] } 53 | 54 | context "with validation checks turned off" do 55 | it "should import valid data" do 56 | assert_difference "Topic.count", +2 do 57 | Topic.import columns, valid_values, validate: false 58 | end 59 | end 60 | 61 | it "should import invalid data" do 62 | assert_difference "Topic.count", +2 do 63 | Topic.import columns, invalid_values, validate: false 64 | end 65 | end 66 | 67 | it 'should raise a specific error if a column does not exist' do 68 | assert_raises ActiveRecord::Import::MissingColumnError do 69 | Topic.import ['foo'], [['bar']], validate: false 70 | end 71 | end 72 | end 73 | 74 | context "with validation checks turned on" do 75 | it "should import valid data" do 76 | assert_difference "Topic.count", +2 do 77 | Topic.import columns, valid_values, validate: true 78 | end 79 | end 80 | 81 | it "should import valid data with on option" do 82 | assert_difference "Topic.count", +2 do 83 | Topic.import columns, valid_values_with_context, validate_with_context: :context_test 84 | end 85 | end 86 | 87 | it "should not import invalid data" do 88 | assert_no_difference "Topic.count" do 89 | Topic.import columns, invalid_values, validate: true 90 | end 91 | end 92 | 93 | it "should import invalid data with on option" do 94 | assert_no_difference "Topic.count" do 95 | Topic.import columns, valid_values, validate_with_context: :context_test 96 | end 97 | end 98 | 99 | it "should report the failed instances" do 100 | results = Topic.import columns, invalid_values, validate: true 101 | assert_equal invalid_values.size, results.failed_instances.size 102 | results.failed_instances.each { |e| assert_kind_of Topic, e } 103 | end 104 | 105 | it "should import valid data when mixed with invalid data" do 106 | assert_difference "Topic.count", +2 do 107 | Topic.import columns, valid_values + invalid_values, validate: true 108 | end 109 | assert_equal 0, Topic.where(title: invalid_values.map(&:first)).count 110 | end 111 | end 112 | end 113 | 114 | context "with :all_or_none option" do 115 | let(:columns) { %w(title author_name) } 116 | let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] } 117 | let(:invalid_values) { [["The RSpec Book", ""], ["Agile+UX", ""]] } 118 | let(:mixed_values) { valid_values + invalid_values } 119 | 120 | context "with validation checks turned on" do 121 | it "should import valid data" do 122 | assert_difference "Topic.count", +2 do 123 | Topic.import columns, valid_values, all_or_none: true 124 | end 125 | end 126 | 127 | it "should not import invalid data" do 128 | assert_no_difference "Topic.count" do 129 | Topic.import columns, invalid_values, all_or_none: true 130 | end 131 | end 132 | 133 | it "should not import valid data when mixed with invalid data" do 134 | assert_no_difference "Topic.count" do 135 | Topic.import columns, mixed_values, all_or_none: true 136 | end 137 | end 138 | 139 | it "should report the failed instances" do 140 | results = Topic.import columns, mixed_values, all_or_none: true 141 | assert_equal invalid_values.size, results.failed_instances.size 142 | results.failed_instances.each { |e| assert_kind_of Topic, e } 143 | end 144 | 145 | it "should report the zero inserts" do 146 | results = Topic.import columns, mixed_values, all_or_none: true 147 | assert_equal 0, results.num_inserts 148 | end 149 | end 150 | end 151 | 152 | context "with :batch_size option" do 153 | it "should import with a single insert" do 154 | assert_difference "Topic.count", +10 do 155 | result = Topic.import Build(10, :topics), batch_size: 10 156 | assert_equal 1, result.num_inserts if Topic.supports_import? 157 | end 158 | end 159 | 160 | it "should import with multiple inserts" do 161 | assert_difference "Topic.count", +10 do 162 | result = Topic.import Build(10, :topics), batch_size: 4 163 | assert_equal 3, result.num_inserts if Topic.supports_import? 164 | end 165 | end 166 | end 167 | 168 | context "with :synchronize option" do 169 | context "synchronizing on new records" do 170 | let(:new_topics) { Build(3, :topics) } 171 | 172 | it "doesn't reload any data (doesn't work)" do 173 | Topic.import new_topics, synchronize: new_topics 174 | if Topic.support_setting_primary_key_of_imported_objects? 175 | assert new_topics.all?(&:persisted?), "Records should have been reloaded" 176 | else 177 | assert new_topics.all?(&:new_record?), "No record should have been reloaded" 178 | end 179 | end 180 | end 181 | 182 | context "synchronizing on new records with explicit conditions" do 183 | let(:new_topics) { Build(3, :topics) } 184 | 185 | it "reloads data for existing in-memory instances" do 186 | Topic.import(new_topics, synchronize: new_topics, synchronize_keys: [:title] ) 187 | assert new_topics.all?(&:persisted?), "Records should have been reloaded" 188 | end 189 | end 190 | 191 | context "synchronizing on destroyed records with explicit conditions" do 192 | let(:new_topics) { Generate(3, :topics) } 193 | 194 | it "reloads data for existing in-memory instances" do 195 | new_topics.each(&:destroy) 196 | Topic.import(new_topics, synchronize: new_topics, synchronize_keys: [:title] ) 197 | assert new_topics.all?(&:persisted?), "Records should have been reloaded" 198 | end 199 | end 200 | end 201 | 202 | context "with an array of unsaved model instances" do 203 | let(:topic) { Build(:topic, title: "The RSpec Book", author_name: "David Chelimsky") } 204 | let(:topics) { Build(9, :topics) } 205 | let(:invalid_topics) { Build(7, :invalid_topics) } 206 | 207 | it "should import records based on those model's attributes" do 208 | assert_difference "Topic.count", +9 do 209 | Topic.import topics 210 | end 211 | 212 | Topic.import [topic] 213 | assert Topic.where(title: "The RSpec Book", author_name: "David Chelimsky").first 214 | end 215 | 216 | it "should not overwrite existing records" do 217 | topic = Generate(:topic, title: "foobar") 218 | assert_no_difference "Topic.count" do 219 | begin 220 | Topic.transaction do 221 | topic.title = "baz" 222 | Topic.import [topic] 223 | end 224 | rescue Exception 225 | # PostgreSQL raises PgError due to key constraints 226 | # I don't know why ActiveRecord doesn't catch these. *sigh* 227 | end 228 | end 229 | assert_equal "foobar", topic.reload.title 230 | end 231 | 232 | context "with validation checks turned on" do 233 | it "should import valid models" do 234 | assert_difference "Topic.count", +9 do 235 | Topic.import topics, validate: true 236 | end 237 | end 238 | 239 | it "should not import invalid models" do 240 | assert_no_difference "Topic.count" do 241 | Topic.import invalid_topics, validate: true 242 | end 243 | end 244 | end 245 | 246 | context "with validation checks turned off" do 247 | it "should import invalid models" do 248 | assert_difference "Topic.count", +7 do 249 | Topic.import invalid_topics, validate: false 250 | end 251 | end 252 | end 253 | end 254 | 255 | context "with an array of columns and an array of unsaved model instances" do 256 | let(:topics) { Build(2, :topics) } 257 | 258 | it "should import records populating the supplied columns with the corresponding model instance attributes" do 259 | assert_difference "Topic.count", +2 do 260 | Topic.import [:author_name, :title], topics 261 | end 262 | 263 | # imported topics should be findable by their imported attributes 264 | assert Topic.where(author_name: topics.first.author_name).first 265 | assert Topic.where(author_name: topics.last.author_name).first 266 | end 267 | 268 | it "should not populate fields for columns not imported" do 269 | topics.first.author_email_address = "zach.dennis@gmail.com" 270 | assert_difference "Topic.count", +2 do 271 | Topic.import [:author_name, :title], topics 272 | end 273 | 274 | assert !Topic.where(author_email_address: "zach.dennis@gmail.com").first 275 | end 276 | end 277 | 278 | context "with an array of columns and an array of values" do 279 | it "should import ids when specified" do 280 | Topic.import [:id, :author_name, :title], [[99, "Bob Jones", "Topic 99"]] 281 | assert_equal 99, Topic.last.id 282 | end 283 | end 284 | 285 | context "ActiveRecord timestamps" do 286 | let(:time) { Chronic.parse("5 minutes ago") } 287 | 288 | context "when the timestamps columns are present" do 289 | setup do 290 | @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc) 291 | ActiveRecord::Base.default_timezone = :utc 292 | Timecop.freeze(time) do 293 | assert_difference "Book.count", +2 do 294 | Book.import %w(title author_name publisher created_at created_on), [["LDAP", "Big Bird", "Del Rey", nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on]] 295 | end 296 | end 297 | @new_book, @existing_book = Book.last 2 298 | end 299 | 300 | it "should set the created_at column for new records" do 301 | assert_in_delta time.to_i, @new_book.created_at.to_i, 1.second 302 | end 303 | 304 | it "should set the created_on column for new records" do 305 | assert_in_delta time.to_i, @new_book.created_on.to_i, 1.second 306 | end 307 | 308 | it "should not set the created_at column for existing records" do 309 | assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.created_at.strftime("%Y:%d") 310 | end 311 | 312 | it "should not set the created_on column for existing records" do 313 | assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.created_on.strftime("%Y:%d") 314 | end 315 | 316 | it "should set the updated_at column for new records" do 317 | assert_in_delta time.to_i, @new_book.updated_at.to_i, 1.second 318 | end 319 | 320 | it "should set the updated_on column for new records" do 321 | assert_in_delta time.to_i, @new_book.updated_on.to_i, 1.second 322 | end 323 | end 324 | 325 | context "when a custom time zone is set" do 326 | setup do 327 | Timecop.freeze(time) do 328 | assert_difference "Book.count", +1 do 329 | Book.import [:title, :author_name, :publisher], [["LDAP", "Big Bird", "Del Rey"]] 330 | end 331 | end 332 | @book = Book.last 333 | end 334 | 335 | it "should set the created_at and created_on timestamps for new records" do 336 | assert_in_delta time.to_i, @book.created_at.to_i, 1.second 337 | assert_in_delta time.to_i, @book.created_on.to_i, 1.second 338 | end 339 | 340 | it "should set the updated_at and updated_on timestamps for new records" do 341 | assert_in_delta time.to_i, @book.updated_at.to_i, 1.second 342 | assert_in_delta time.to_i, @book.updated_on.to_i, 1.second 343 | end 344 | end 345 | end 346 | 347 | context "importing with database reserved words" do 348 | let(:group) { Build(:group, order: "superx") } 349 | 350 | it "should import just fine" do 351 | assert_difference "Group.count", +1 do 352 | Group.import [group] 353 | end 354 | assert_equal "superx", Group.first.order 355 | end 356 | end 357 | 358 | context "importing a datetime field" do 359 | it "should import a date with YYYY/MM/DD format just fine" do 360 | Topic.import [:author_name, :title, :last_read], [["Bob Jones", "Topic 2", "2010/05/14"]] 361 | assert_equal "2010/05/14".to_date, Topic.last.last_read.to_date 362 | end 363 | end 364 | 365 | context "importing through an association scope" do 366 | [true, false].each do |bool| 367 | context "when validation is " + (bool ? "enabled" : "disabled") do 368 | it "should automatically set the foreign key column" do 369 | books = [["David Chelimsky", "The RSpec Book"], ["Chad Fowler", "Rails Recipes"]] 370 | topic = FactoryGirl.create :topic 371 | topic.books.import [:author_name, :title], books, validate: bool 372 | assert_equal 2, topic.books.count 373 | assert topic.books.all? { |book| book.topic_id == topic.id } 374 | end 375 | end 376 | end 377 | 378 | it "works importing models" do 379 | topic = FactoryGirl.create :topic 380 | books = [ 381 | Book.new(author_name: "Author #1", title: "Book #1"), 382 | Book.new(author_name: "Author #2", title: "Book #2"), 383 | ] 384 | topic.books.import books 385 | assert_equal 2, topic.books.count 386 | assert topic.books.detect { |b| b.title == "Book #1" && b.author_name == "Author #1" } 387 | assert topic.books.detect { |b| b.title == "Book #2" && b.author_name == "Author #2" } 388 | end 389 | 390 | it "works importing array of columns and values" do 391 | topic = FactoryGirl.create :topic 392 | topic.books.import [:author_name, :title], [["Author #1", "Book #1"], ["Author #2", "Book #2"]] 393 | assert_equal 2, topic.books.count 394 | assert topic.books.detect { |b| b.title == "Book #1" && b.author_name == "Author #1" } 395 | assert topic.books.detect { |b| b.title == "Book #2" && b.author_name == "Author #2" } 396 | end 397 | end 398 | 399 | context 'When importing models with Enum fields' do 400 | it 'should be able to import enum fields' do 401 | Book.delete_all if Book.count > 0 402 | books = [ 403 | Book.new(author_name: "Foo", title: "Baz", status: 0), 404 | Book.new(author_name: "Foo2", title: "Baz2", status: 1), 405 | ] 406 | Book.import books 407 | assert_equal 2, Book.count 408 | 409 | if ENV['AR_VERSION'].to_i >= 5.0 410 | assert_equal 'draft', Book.first.read_attribute('status') 411 | assert_equal 'published', Book.last.read_attribute('status') 412 | else 413 | assert_equal 0, Book.first.read_attribute('status') 414 | assert_equal 1, Book.last.read_attribute('status') 415 | end 416 | end 417 | 418 | it 'should be able to import enum fields with default value' do 419 | Book.delete_all if Book.count > 0 420 | books = [ 421 | Book.new(author_name: "Foo", title: "Baz") 422 | ] 423 | Book.import books 424 | assert_equal 1, Book.count 425 | 426 | if ENV['AR_VERSION'].to_i >= 5.0 427 | assert_equal 'draft', Book.first.read_attribute('status') 428 | else 429 | assert_equal 0, Book.first.read_attribute('status') 430 | end 431 | end 432 | 433 | if ENV['AR_VERSION'].to_f > 4.1 434 | it 'should be able to import enum fields by name' do 435 | Book.delete_all if Book.count > 0 436 | books = [ 437 | Book.new(author_name: "Foo", title: "Baz", status: :draft), 438 | Book.new(author_name: "Foo2", title: "Baz2", status: :published), 439 | ] 440 | Book.import books 441 | assert_equal 2, Book.count 442 | 443 | if ENV['AR_VERSION'].to_i >= 5.0 444 | assert_equal 'draft', Book.first.read_attribute('status') 445 | assert_equal 'published', Book.last.read_attribute('status') 446 | else 447 | assert_equal 0, Book.first.read_attribute('status') 448 | assert_equal 1, Book.last.read_attribute('status') 449 | end 450 | end 451 | end 452 | end 453 | 454 | describe "importing when model has default_scope" do 455 | it "doesn't import the default scope values" do 456 | assert_difference "Widget.unscoped.count", +2 do 457 | Widget.import [:w_id], [[1], [2]] 458 | end 459 | default_scope_value = Widget.scope_attributes[:active] 460 | assert_not_equal default_scope_value, Widget.unscoped.find_by_w_id(1) 461 | assert_not_equal default_scope_value, Widget.unscoped.find_by_w_id(2) 462 | end 463 | 464 | it "imports columns that are a part of the default scope using the value specified" do 465 | assert_difference "Widget.unscoped.count", +2 do 466 | Widget.import [:w_id, :active], [[1, true], [2, false]] 467 | end 468 | assert_not_equal true, Widget.unscoped.find_by_w_id(1) 469 | assert_not_equal false, Widget.unscoped.find_by_w_id(2) 470 | end 471 | end 472 | 473 | describe "importing serialized fields" do 474 | it "imports values for serialized fields" do 475 | assert_difference "Widget.unscoped.count", +1 do 476 | Widget.import [:w_id, :data], [[1, { a: :b }]] 477 | end 478 | assert_equal({ a: :b }, Widget.find_by_w_id(1).data) 479 | end 480 | 481 | if ENV['AR_VERSION'].to_f >= 3.1 482 | let(:data) { { a: :b } } 483 | it "imports values for serialized JSON fields" do 484 | assert_difference "Widget.unscoped.count", +1 do 485 | Widget.import [:w_id, :json_data], [[9, data]] 486 | end 487 | assert_equal(data.as_json, Widget.find_by_w_id(9).json_data) 488 | end 489 | end 490 | end 491 | 492 | describe "#import!" do 493 | let(:columns) { %w(title author_name) } 494 | let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] } 495 | let(:invalid_values) { [["Rails Recipes", "Chad Fowler"], ["The RSpec Book", ""], ["Agile+UX", ""]] } 496 | 497 | context "with invalid data" do 498 | it "should raise ActiveRecord::RecordInvalid" do 499 | assert_no_difference "Topic.count" do 500 | assert_raise ActiveRecord::RecordInvalid do 501 | Topic.import! columns, invalid_values 502 | end 503 | end 504 | end 505 | end 506 | 507 | context "with valid data" do 508 | it "should import data" do 509 | assert_difference "Topic.count", +2 do 510 | Topic.import! columns, valid_values 511 | end 512 | end 513 | end 514 | end 515 | end 516 | -------------------------------------------------------------------------------- /lib/activerecord-import/import.rb: -------------------------------------------------------------------------------- 1 | require "ostruct" 2 | 3 | module ActiveRecord::Import::ConnectionAdapters; end 4 | 5 | module ActiveRecord::Import #:nodoc: 6 | Result = Struct.new(:failed_instances, :num_inserts, :ids) 7 | 8 | module ImportSupport #:nodoc: 9 | def supports_import? #:nodoc: 10 | true 11 | end 12 | end 13 | 14 | module OnDuplicateKeyUpdateSupport #:nodoc: 15 | def supports_on_duplicate_key_update? #:nodoc: 16 | true 17 | end 18 | end 19 | 20 | class MissingColumnError < StandardError 21 | def initialize(name, index) 22 | super "Missing column for value <#{name}> at index #{index}" 23 | end 24 | end 25 | end 26 | 27 | class ActiveRecord::Associations::CollectionProxy 28 | def import(*args, &block) 29 | @association.import(*args, &block) 30 | end 31 | end 32 | 33 | class ActiveRecord::Associations::CollectionAssociation 34 | def import(*args, &block) 35 | unless owner.persisted? 36 | raise ActiveRecord::RecordNotSaved, "You cannot call import unless the parent is saved" 37 | end 38 | 39 | options = args.last.is_a?(Hash) ? args.pop : {} 40 | 41 | model_klass = reflection.klass 42 | symbolized_foreign_key = reflection.foreign_key.to_sym 43 | symbolized_column_names = model_klass.column_names.map(&:to_sym) 44 | 45 | owner_primary_key = owner.class.primary_key 46 | owner_primary_key_value = owner.send(owner_primary_key) 47 | 48 | # assume array of model objects 49 | if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base) 50 | if args.length == 2 51 | models = args.last 52 | column_names = args.first 53 | else 54 | models = args.first 55 | column_names = symbolized_column_names 56 | end 57 | 58 | unless symbolized_column_names.include?(symbolized_foreign_key) 59 | column_names << symbolized_foreign_key 60 | end 61 | 62 | models.each do |m| 63 | m.public_send "#{symbolized_foreign_key}=", owner_primary_key_value 64 | end 65 | 66 | return model_klass.import column_names, models, options 67 | 68 | # supports empty array 69 | elsif args.last.is_a?( Array ) && args.last.empty? 70 | return ActiveRecord::Import::Result.new([], 0, []) if args.last.empty? 71 | 72 | # supports 2-element array and array 73 | elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array ) 74 | column_names, array_of_attributes = args 75 | symbolized_column_names = column_names.map(&:to_s) 76 | 77 | if symbolized_column_names.include?(symbolized_foreign_key) 78 | index = symbolized_column_names.index(symbolized_foreign_key) 79 | array_of_attributes.each { |attrs| attrs[index] = owner_primary_key_value } 80 | else 81 | column_names << symbolized_foreign_key 82 | array_of_attributes.each { |attrs| attrs << owner_primary_key_value } 83 | end 84 | 85 | return model_klass.import column_names, array_of_attributes, options 86 | else 87 | raise ArgumentError, "Invalid arguments!" 88 | end 89 | end 90 | end 91 | 92 | class ActiveRecord::Base 93 | class << self 94 | # use tz as set in ActiveRecord::Base 95 | tproc = lambda do 96 | ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now 97 | end 98 | 99 | AREXT_RAILS_COLUMNS = { 100 | create: { "created_on" => tproc, 101 | "created_at" => tproc }, 102 | update: { "updated_on" => tproc, 103 | "updated_at" => tproc } 104 | }.freeze 105 | AREXT_RAILS_COLUMN_NAMES = AREXT_RAILS_COLUMNS[:create].keys + AREXT_RAILS_COLUMNS[:update].keys 106 | 107 | # Returns true if the current database connection adapter 108 | # supports import functionality, otherwise returns false. 109 | def supports_import?(*args) 110 | connection.respond_to?(:supports_import?) && connection.supports_import?(*args) 111 | end 112 | 113 | # Returns true if the current database connection adapter 114 | # supports on duplicate key update functionality, otherwise 115 | # returns false. 116 | def supports_on_duplicate_key_update? 117 | connection.supports_on_duplicate_key_update? 118 | end 119 | 120 | # returns true if the current database connection adapter 121 | # supports setting the primary key of bulk imported models, otherwise 122 | # returns false 123 | def support_setting_primary_key_of_imported_objects? 124 | connection.respond_to?(:support_setting_primary_key_of_imported_objects?) && connection.support_setting_primary_key_of_imported_objects? 125 | end 126 | 127 | # Imports a collection of values to the database. 128 | # 129 | # This is more efficient than using ActiveRecord::Base#create or 130 | # ActiveRecord::Base#save multiple times. This method works well if 131 | # you want to create more than one record at a time and do not care 132 | # about having ActiveRecord objects returned for each record 133 | # inserted. 134 | # 135 | # This can be used with or without validations. It does not utilize 136 | # the ActiveRecord::Callbacks during creation/modification while 137 | # performing the import. 138 | # 139 | # == Usage 140 | # Model.import array_of_models 141 | # Model.import column_names, array_of_values 142 | # Model.import column_names, array_of_values, options 143 | # 144 | # ==== Model.import array_of_models 145 | # 146 | # With this form you can call _import_ passing in an array of model 147 | # objects that you want updated. 148 | # 149 | # ==== Model.import column_names, array_of_values 150 | # 151 | # The first parameter +column_names+ is an array of symbols or 152 | # strings which specify the columns that you want to update. 153 | # 154 | # The second parameter, +array_of_values+, is an array of 155 | # arrays. Each subarray is a single set of values for a new 156 | # record. The order of values in each subarray should match up to 157 | # the order of the +column_names+. 158 | # 159 | # ==== Model.import column_names, array_of_values, options 160 | # 161 | # The first two parameters are the same as the above form. The third 162 | # parameter, +options+, is a hash. This is optional. Please see 163 | # below for what +options+ are available. 164 | # 165 | # == Options 166 | # * +validate+ - true|false, tells import whether or not to use 167 | # ActiveRecord validations. Validations are enforced by default. 168 | # * +ignore+ - true|false, tells import to use MySQL's INSERT IGNORE 169 | # to discard records that contain duplicate keys. 170 | # * +on_duplicate_key_ignore+ - true|false, tells import to use 171 | # Postgres 9.5+ ON CONFLICT DO NOTHING. 172 | # * +on_duplicate_key_update+ - an Array or Hash, tells import to 173 | # use MySQL's ON DUPLICATE KEY UPDATE or Postgres 9.5+ ON CONFLICT 174 | # DO UPDATE ability. See On Duplicate Key Update below. 175 | # * +synchronize+ - an array of ActiveRecord instances for the model 176 | # that you are currently importing data into. This synchronizes 177 | # existing model instances in memory with updates from the import. 178 | # * +timestamps+ - true|false, tells import to not add timestamps 179 | # (if false) even if record timestamps is disabled in ActiveRecord::Base 180 | # * +recursive+ - true|false, tells import to import all has_many/has_one 181 | # associations if the adapter supports setting the primary keys of the 182 | # newly imported objects. 183 | # * +batch_size+ - an integer value to specify the max number of records to 184 | # include per insert. Defaults to the total number of records to import. 185 | # 186 | # == Examples 187 | # class BlogPost < ActiveRecord::Base ; end 188 | # 189 | # # Example using array of model objects 190 | # posts = [ BlogPost.new author_name: 'Zach Dennis', title: 'AREXT', 191 | # BlogPost.new author_name: 'Zach Dennis', title: 'AREXT2', 192 | # BlogPost.new author_name: 'Zach Dennis', title: 'AREXT3' ] 193 | # BlogPost.import posts 194 | # 195 | # # Example using column_names and array_of_values 196 | # columns = [ :author_name, :title ] 197 | # values = [ [ 'zdennis', 'test post' ], [ 'jdoe', 'another test post' ] ] 198 | # BlogPost.import columns, values 199 | # 200 | # # Example using column_names, array_of_value and options 201 | # columns = [ :author_name, :title ] 202 | # values = [ [ 'zdennis', 'test post' ], [ 'jdoe', 'another test post' ] ] 203 | # BlogPost.import( columns, values, validate: false ) 204 | # 205 | # # Example synchronizing existing instances in memory 206 | # post = BlogPost.where(author_name: 'zdennis').first 207 | # puts post.author_name # => 'zdennis' 208 | # columns = [ :author_name, :title ] 209 | # values = [ [ 'yoda', 'test post' ] ] 210 | # BlogPost.import posts, synchronize: [ post ] 211 | # puts post.author_name # => 'yoda' 212 | # 213 | # # Example synchronizing unsaved/new instances in memory by using a uniqued imported field 214 | # posts = [BlogPost.new(title: "Foo"), BlogPost.new(title: "Bar")] 215 | # BlogPost.import posts, synchronize: posts, synchronize_keys: [:title] 216 | # puts posts.first.persisted? # => true 217 | # 218 | # == On Duplicate Key Update (MySQL) 219 | # 220 | # The :on_duplicate_key_update option can be either an Array or a Hash. 221 | # 222 | # ==== Using an Array 223 | # 224 | # The :on_duplicate_key_update option can be an array of column 225 | # names. The column names are the only fields that are updated if 226 | # a duplicate record is found. Below is an example: 227 | # 228 | # BlogPost.import columns, values, on_duplicate_key_update: [ :date_modified, :content, :author ] 229 | # 230 | # ==== Using A Hash 231 | # 232 | # The :on_duplicate_key_update option can be a hash of column names 233 | # to model attribute name mappings. This gives you finer grained 234 | # control over what fields are updated with what attributes on your 235 | # model. Below is an example: 236 | # 237 | # BlogPost.import columns, attributes, on_duplicate_key_update: { title: :title } 238 | # 239 | # == On Duplicate Key Update (Postgres 9.5+) 240 | # 241 | # The :on_duplicate_key_update option can be an Array or a Hash with up to 242 | # two attributes, :conflict_target or :constraint_name and :columns. 243 | # 244 | # ==== Using an Array 245 | # 246 | # The :on_duplicate_key_update option can be an array of column 247 | # names. This option only handles inserts that conflict with the 248 | # primary key. If a table does not have a primary key, this will 249 | # not work. The column names are the only fields that are updated 250 | # if a duplicate record is found. Below is an example: 251 | # 252 | # BlogPost.import columns, values, on_duplicate_key_update: [ :date_modified, :content, :author ] 253 | # 254 | # ==== Using a Hash 255 | # 256 | # The :on_duplicate_update option can be a hash with up to two attributes, 257 | # :conflict_target or constraint_name, and :columns. Unlike MySQL, Postgres 258 | # requires the conflicting constraint to be explicitly specified. Using this 259 | # option allows you to specify a constraint other than the primary key. 260 | # 261 | # ====== :conflict_target 262 | # 263 | # The :conflict_target attribute specifies the columns that make up the 264 | # conflicting unique constraint and can be a single column or an array of 265 | # column names. This attribute is ignored if :constraint_name is included, 266 | # but it is the preferred method of identifying a constraint. It will 267 | # default to the primary key. Below is an example: 268 | # 269 | # BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [:author_id, :slug], columns: [ :date_modified ] } 270 | # 271 | # ====== :constraint_name 272 | # 273 | # The :constraint_name attribute explicitly identifies the conflicting 274 | # unique index by name. Postgres documentation discourages using this method 275 | # of identifying an index unless absolutely necessary. Below is an example: 276 | # 277 | # BlogPost.import columns, values, on_duplicate_key_update: { constraint_name: :blog_posts_pkey, columns: [ :date_modified ] } 278 | # 279 | # ====== :columns 280 | # 281 | # The :columns attribute can be either an Array or a Hash. 282 | # 283 | # ======== Using an Array 284 | # 285 | # The :columns attribute can be an array of column names. The column names 286 | # are the only fields that are updated if a duplicate record is found. 287 | # Below is an example: 288 | # 289 | # BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: :slug, columns: [ :date_modified, :content, :author ] } 290 | # 291 | # ======== Using a Hash 292 | # 293 | # The :columns option can be a hash of column names to model attribute name 294 | # mappings. This gives you finer grained control over what fields are updated 295 | # with what attributes on your model. Below is an example: 296 | # 297 | # BlogPost.import columns, attributes, on_duplicate_key_update: { conflict_target: :slug, columns: { title: :title } } 298 | # 299 | # = Returns 300 | # This returns an object which responds to +failed_instances+ and +num_inserts+. 301 | # * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed. 302 | # * num_inserts - the number of insert statements it took to import the data 303 | # * ids - the primary keys of the imported ids, if the adpater supports it, otherwise and empty array. 304 | def import(*args) 305 | if args.first.is_a?( Array ) && args.first.first.is_a?(ActiveRecord::Base) 306 | options = {} 307 | options.merge!( args.pop ) if args.last.is_a?(Hash) 308 | 309 | models = args.first 310 | import_helper(models, options) 311 | else 312 | import_helper(*args) 313 | end 314 | end 315 | 316 | # Imports a collection of values if all values are valid. Import fails at the 317 | # first encountered validation error and raises ActiveRecord::RecordInvalid 318 | # with the failed instance. 319 | def import!(*args) 320 | options = args.last.is_a?( Hash ) ? args.pop : {} 321 | options[:validate] = true 322 | options[:raise_error] = true 323 | 324 | import(*args, options) 325 | end 326 | 327 | def import_helper( *args ) 328 | options = { validate: true, timestamps: true, primary_key: primary_key } 329 | options.merge!( args.pop ) if args.last.is_a? Hash 330 | 331 | # Don't modify incoming arguments 332 | if options[:on_duplicate_key_update] 333 | options[:on_duplicate_key_update] = options[:on_duplicate_key_update].dup 334 | end 335 | 336 | is_validating = options[:validate] 337 | is_validating = true unless options[:validate_with_context].nil? 338 | 339 | # assume array of model objects 340 | if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base) 341 | if args.length == 2 342 | models = args.last 343 | column_names = args.first 344 | else 345 | models = args.first 346 | column_names = self.column_names.dup 347 | end 348 | 349 | array_of_attributes = models.map do |model| 350 | # this next line breaks sqlite.so with a segmentation fault 351 | # if model.new_record? || options[:on_duplicate_key_update] 352 | column_names.map do |name| 353 | name = name.to_s 354 | if respond_to?(:defined_enums) && defined_enums.key?(name) # ActiveRecord 5 355 | model.read_attribute(name) 356 | else 357 | model.read_attribute_before_type_cast(name) 358 | end 359 | end 360 | # end 361 | end 362 | # supports empty array 363 | elsif args.last.is_a?( Array ) && args.last.empty? 364 | return ActiveRecord::Import::Result.new([], 0, []) if args.last.empty? 365 | # supports 2-element array and array 366 | elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array ) 367 | column_names, array_of_attributes = args 368 | else 369 | raise ArgumentError, "Invalid arguments!" 370 | end 371 | 372 | # dup the passed in array so we don't modify it unintentionally 373 | column_names = column_names.dup 374 | array_of_attributes = array_of_attributes.dup 375 | 376 | # Force the primary key col into the insert if it's not 377 | # on the list and we are using a sequence and stuff a nil 378 | # value for it into each row so the sequencer will fire later 379 | if !column_names.include?(primary_key) && connection.prefetch_primary_key? && sequence_name 380 | column_names << primary_key 381 | array_of_attributes.each { |a| a << nil } 382 | end 383 | 384 | # record timestamps unless disabled in ActiveRecord::Base 385 | if record_timestamps && options.delete( :timestamps ) 386 | add_special_rails_stamps column_names, array_of_attributes, options 387 | end 388 | 389 | return_obj = if is_validating 390 | import_with_validations( column_names, array_of_attributes, options ) 391 | else 392 | (num_inserts, ids) = import_without_validations_or_callbacks( column_names, array_of_attributes, options ) 393 | ActiveRecord::Import::Result.new([], num_inserts, ids) 394 | end 395 | 396 | if options[:synchronize] 397 | sync_keys = options[:synchronize_keys] || [primary_key] 398 | synchronize( options[:synchronize], sync_keys) 399 | end 400 | return_obj.num_inserts = 0 if return_obj.num_inserts.nil? 401 | 402 | # if we have ids, then set the id on the models and mark the models as clean. 403 | if support_setting_primary_key_of_imported_objects? 404 | set_ids_and_mark_clean(models, return_obj) 405 | 406 | # if there are auto-save associations on the models we imported that are new, import them as well 407 | import_associations(models, options.dup) if options[:recursive] 408 | end 409 | 410 | return_obj 411 | end 412 | 413 | # TODO import_from_table needs to be implemented. 414 | def import_from_table( options ) # :nodoc: 415 | end 416 | 417 | # Imports the passed in +column_names+ and +array_of_attributes+ 418 | # given the passed in +options+ Hash with validations. Returns an 419 | # object with the methods +failed_instances+ and +num_inserts+. 420 | # +failed_instances+ is an array of instances that failed validations. 421 | # +num_inserts+ is the number of inserts it took to import the data. See 422 | # ActiveRecord::Base.import for more information on 423 | # +column_names+, +array_of_attributes+ and +options+. 424 | def import_with_validations( column_names, array_of_attributes, options = {} ) 425 | failed_instances = [] 426 | 427 | # create instances for each of our column/value sets 428 | arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes ) 429 | 430 | # keep track of the instance and the position it is currently at. if this fails 431 | # validation we'll use the index to remove it from the array_of_attributes 432 | arr.each_with_index do |hsh, i| 433 | instance = new do |model| 434 | hsh.each_pair { |k, v| model[k] = v } 435 | end 436 | 437 | next if instance.valid?(options[:validate_with_context]) 438 | raise(ActiveRecord::RecordInvalid, instance) if options[:raise_error] 439 | array_of_attributes[i] = nil 440 | failed_instances << instance 441 | end 442 | array_of_attributes.compact! 443 | 444 | num_inserts, ids = if array_of_attributes.empty? || options[:all_or_none] && failed_instances.any? 445 | [0, []] 446 | else 447 | import_without_validations_or_callbacks( column_names, array_of_attributes, options ) 448 | end 449 | ActiveRecord::Import::Result.new(failed_instances, num_inserts, ids) 450 | end 451 | 452 | # Imports the passed in +column_names+ and +array_of_attributes+ 453 | # given the passed in +options+ Hash. This will return the number 454 | # of insert operations it took to create these records without 455 | # validations or callbacks. See ActiveRecord::Base.import for more 456 | # information on +column_names+, +array_of_attributes_ and 457 | # +options+. 458 | def import_without_validations_or_callbacks( column_names, array_of_attributes, options = {} ) 459 | column_names = column_names.map(&:to_sym) 460 | scope_columns, scope_values = scope_attributes.to_a.transpose 461 | 462 | unless scope_columns.blank? 463 | scope_columns.zip(scope_values).each do |name, value| 464 | name_as_sym = name.to_sym 465 | next if column_names.include?(name_as_sym) 466 | 467 | is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class) 468 | value = value.first if is_sti 469 | 470 | column_names << name_as_sym 471 | array_of_attributes.each { |attrs| attrs << value } 472 | end 473 | end 474 | 475 | columns = column_names.each_with_index.map do |name, i| 476 | column = columns_hash[name.to_s] 477 | 478 | raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil? 479 | 480 | column 481 | end 482 | 483 | columns_sql = "(#{column_names.map { |name| connection.quote_column_name(name) }.join(',')})" 484 | insert_sql = "INSERT #{options[:ignore] ? 'IGNORE ' : ''}INTO #{quoted_table_name} #{columns_sql} VALUES " 485 | values_sql = values_sql_for_columns_and_attributes(columns, array_of_attributes) 486 | 487 | number_inserted = 0 488 | ids = [] 489 | if supports_import? 490 | # generate the sql 491 | post_sql_statements = connection.post_sql_statements( quoted_table_name, options ) 492 | 493 | batch_size = options[:batch_size] || values_sql.size 494 | values_sql.each_slice(batch_size) do |batch_values| 495 | # perform the inserts 496 | result = connection.insert_many( [insert_sql, post_sql_statements].flatten, 497 | batch_values, 498 | "#{self.class.name} Create Many Without Validations Or Callbacks" ) 499 | number_inserted += result[0] 500 | ids += result[1] 501 | end 502 | else 503 | values_sql.each do |values| 504 | connection.execute(insert_sql + values) 505 | number_inserted += 1 506 | end 507 | end 508 | [number_inserted, ids] 509 | end 510 | 511 | private 512 | 513 | def set_ids_and_mark_clean(models, import_result) 514 | return if models.nil? 515 | import_result.ids.each_with_index do |id, index| 516 | model = models[index] 517 | model.id = id.to_i 518 | if model.respond_to?(:clear_changes_information) # Rails 4.0 and higher 519 | model.clear_changes_information 520 | else # Rails 3.1 521 | model.instance_variable_get(:@changed_attributes).clear 522 | end 523 | model.instance_variable_set(:@new_record, false) 524 | end 525 | end 526 | 527 | def import_associations(models, options) 528 | # now, for all the dirty associations, collect them into a new set of models, then recurse. 529 | # notes: 530 | # does not handle associations that reference themselves 531 | # should probably take a hash to associations to follow. 532 | associated_objects_by_class = {} 533 | models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) } 534 | 535 | # :on_duplicate_key_update not supported for associations 536 | options.delete(:on_duplicate_key_update) 537 | 538 | associated_objects_by_class.each_value do |associations| 539 | associations.each_value do |associated_records| 540 | associated_records.first.class.import(associated_records, options) unless associated_records.empty? 541 | end 542 | end 543 | end 544 | 545 | # We are eventually going to call Class.import so we build up a hash 546 | # of class => objects to import. 547 | def find_associated_objects_for_import(associated_objects_by_class, model) 548 | associated_objects_by_class[model.class.name] ||= {} 549 | 550 | association_reflections = 551 | model.class.reflect_on_all_associations(:has_one) + 552 | model.class.reflect_on_all_associations(:has_many) 553 | association_reflections.each do |association_reflection| 554 | associated_objects_by_class[model.class.name][association_reflection.name] ||= [] 555 | 556 | association = model.association(association_reflection.name) 557 | association.loaded! 558 | 559 | # Wrap target in an array if not already 560 | association = Array(association.target) 561 | 562 | changed_objects = association.select { |a| a.new_record? || a.changed? } 563 | changed_objects.each do |child| 564 | child.public_send("#{association_reflection.foreign_key}=", model.id) 565 | # For polymorphic associations 566 | association_reflection.type.try do |type| 567 | child.public_send("#{type}=", model.class.name) 568 | end 569 | end 570 | associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects 571 | end 572 | associated_objects_by_class 573 | end 574 | 575 | # Returns SQL the VALUES for an INSERT statement given the passed in +columns+ 576 | # and +array_of_attributes+. 577 | def values_sql_for_columns_and_attributes(columns, array_of_attributes) # :nodoc: 578 | # connection gets called a *lot* in this high intensity loop. 579 | # Reuse the same one w/in the loop, otherwise it would keep being re-retreived (= lots of time for large imports) 580 | connection_memo = connection 581 | array_of_attributes.map do |arr| 582 | my_values = arr.each_with_index.map do |val, j| 583 | column = columns[j] 584 | 585 | # be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly 586 | if val.nil? && column.name == primary_key && !sequence_name.blank? 587 | connection_memo.next_value_for_sequence(sequence_name) 588 | elsif column 589 | if respond_to?(:type_caster) && type_caster.respond_to?(:type_cast_for_database) # Rails 5.0 and higher 590 | connection_memo.quote(type_caster.type_cast_for_database(column.name, val)) 591 | elsif column.respond_to?(:type_cast_from_user) # Rails 4.2 and higher 592 | connection_memo.quote(column.type_cast_from_user(val), column) 593 | else # Rails 3.1, 3.2, 4.0 and 4.1 594 | if serialized_attributes.include?(column.name) 595 | val = serialized_attributes[column.name].dump(val) 596 | end 597 | connection_memo.quote(column.type_cast(val), column) 598 | end 599 | end 600 | end 601 | "(#{my_values.join(',')})" 602 | end 603 | end 604 | 605 | def add_special_rails_stamps( column_names, array_of_attributes, options ) 606 | AREXT_RAILS_COLUMNS[:create].each_pair do |key, blk| 607 | next unless self.column_names.include?(key) 608 | value = blk.call 609 | index = column_names.index(key) || column_names.index(key.to_sym) 610 | if index 611 | # replace every instance of the array of attributes with our value 612 | array_of_attributes.each { |arr| arr[index] = value if arr[index].nil? } 613 | else 614 | column_names << key 615 | array_of_attributes.each { |arr| arr << value } 616 | end 617 | end 618 | 619 | AREXT_RAILS_COLUMNS[:update].each_pair do |key, blk| 620 | next unless self.column_names.include?(key) 621 | value = blk.call 622 | index = column_names.index(key) || column_names.index(key.to_sym) 623 | if index 624 | # replace every instance of the array of attributes with our value 625 | array_of_attributes.each { |arr| arr[index] = value } 626 | else 627 | column_names << key 628 | array_of_attributes.each { |arr| arr << value } 629 | end 630 | 631 | if supports_on_duplicate_key_update? 632 | connection.add_column_for_on_duplicate_key_update(key, options) 633 | end 634 | end 635 | end 636 | 637 | # Returns an Array of Hashes for the passed in +column_names+ and +array_of_attributes+. 638 | def validations_array_for_column_names_and_attributes( column_names, array_of_attributes ) # :nodoc: 639 | array_of_attributes.map do |attributes| 640 | Hash[attributes.each_with_index.map { |attr, c| [column_names[c], attr] }] 641 | end 642 | end 643 | end 644 | end 645 | --------------------------------------------------------------------------------