├── .gitignore ├── .travis.yml ├── Appraisals ├── CHANGELOG.md ├── Gemfile ├── MIT-LICENSE ├── README.md ├── Rakefile ├── bulk_insert.gemspec ├── ci └── 000-prepare-database ├── gemfiles ├── rails_3.gemfile ├── rails_4.gemfile ├── rails_5.gemfile └── rails_6.gemfile ├── lib ├── bulk_insert.rb └── bulk_insert │ ├── statement_adapters.rb │ ├── statement_adapters │ ├── base_adapter.rb │ ├── generic_adapter.rb │ ├── mysql_adapter.rb │ ├── postgresql_adapter.rb │ └── sqlite_adapter.rb │ ├── version.rb │ └── worker.rb └── test ├── bulk_insert └── worker_test.rb ├── bulk_insert_test.rb ├── connection_mocks.rb ├── dummy ├── README.rdoc ├── Rakefile ├── app │ ├── assets │ │ ├── config │ │ │ └── manifest.js │ │ ├── images │ │ │ └── .keep │ │ ├── javascripts │ │ │ └── application.js │ │ └── stylesheets │ │ │ └── application.css │ ├── controllers │ │ ├── application_controller.rb │ │ └── concerns │ │ │ └── .keep │ ├── helpers │ │ └── application_helper.rb │ ├── mailers │ │ └── .keep │ ├── models │ │ ├── .keep │ │ ├── concerns │ │ │ └── .keep │ │ └── testing.rb │ └── views │ │ └── layouts │ │ └── application.html.erb ├── bin │ ├── bundle │ ├── rails │ ├── rake │ └── setup ├── config.ru ├── config │ ├── application.rb │ ├── boot.rb │ ├── database.yml │ ├── environment.rb │ ├── environments │ │ ├── development.rb │ │ ├── production.rb │ │ └── test.rb │ ├── initializers │ │ ├── assets.rb │ │ ├── backtrace_silencers.rb │ │ ├── cookies_serializer.rb │ │ ├── filter_parameter_logging.rb │ │ ├── inflections.rb │ │ ├── mime_types.rb │ │ ├── session_store.rb │ │ └── wrap_parameters.rb │ ├── locales │ │ └── en.yml │ ├── routes.rb │ └── secrets.yml ├── db │ ├── migrate │ │ ├── 20151008181535_create_testings.rb │ │ └── 20151028194232_add_default_value.rb │ └── schema.rb ├── lib │ └── assets │ │ └── .keep ├── log │ └── .keep └── public │ ├── 404.html │ ├── 422.html │ ├── 500.html │ └── favicon.ico └── test_helper.rb /.gitignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | .bundle/ 3 | gemfiles/*.lock 4 | log/*.log 5 | pkg/ 6 | test/dummy/db/*.sqlite3 7 | test/dummy/db/*.sqlite3-journal 8 | test/dummy/log/*.log 9 | test/dummy/tmp/ 10 | test/dummy/.sass-cache 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | arch: amd64 3 | os: linux 4 | branch: 5 | - master 6 | # https://docs.travis-ci.com/user/database-setup/#mysql 7 | services: 8 | - mysql 9 | - postgresql 10 | 11 | rvm: 12 | - 2.3 13 | - 2.4 14 | - 2.5 15 | - 2.6 16 | - 2.7 17 | - ruby-head 18 | gemfile: 19 | - gemfiles/rails_5.gemfile 20 | - gemfiles/rails_6.gemfile 21 | env: 22 | - DB_ADAPTER=sqlite 23 | - DB_ADAPTER=mysql 24 | - DB_ADAPTER=postgresql 25 | matrix: 26 | allow_failures: 27 | - rvm: ruby-head 28 | include: 29 | - rvm: 2.2 30 | gemfile: gemfiles/rails_3.gemfile 31 | env: DB_ADAPTER=sqlite 32 | - rvm: 2.2 33 | gemfile: gemfiles/rails_3.gemfile 34 | env: DB_ADAPTER=mysql 35 | - rvm: 2.2 36 | gemfile: gemfiles/rails_3.gemfile 37 | env: DB_ADAPTER=postgresql 38 | - rvm: 2.2 39 | gemfile: gemfiles/rails_4.gemfile 40 | env: DB_ADAPTER=sqlite 41 | - rvm: 2.2 42 | gemfile: gemfiles/rails_4.gemfile 43 | env: DB_ADAPTER=mysql 44 | - rvm: 2.2 45 | gemfile: gemfiles/rails_4.gemfile 46 | env: DB_ADAPTER=postgresql 47 | - rvm: 2.3 48 | gemfile: gemfiles/rails_3.gemfile 49 | env: DB_ADAPTER=sqlite 50 | - rvm: 2.3 51 | gemfile: gemfiles/rails_3.gemfile 52 | env: DB_ADAPTER=mysql 53 | - rvm: 2.3 54 | gemfile: gemfiles/rails_3.gemfile 55 | env: DB_ADAPTER=postgresql 56 | - rvm: 2.3 57 | gemfile: gemfiles/rails_4.gemfile 58 | env: DB_ADAPTER=sqlite 59 | - rvm: 2.3 60 | gemfile: gemfiles/rails_4.gemfile 61 | env: DB_ADAPTER=mysql 62 | - rvm: 2.3 63 | gemfile: gemfiles/rails_4.gemfile 64 | env: DB_ADAPTER=postgresql 65 | - rvm: 2.4 66 | gemfile: gemfiles/rails_4.gemfile 67 | env: DB_ADAPTER=sqlite 68 | - rvm: 2.4 69 | gemfile: gemfiles/rails_4.gemfile 70 | env: DB_ADAPTER=mysql 71 | - rvm: 2.4 72 | gemfile: gemfiles/rails_4.gemfile 73 | env: DB_ADAPTER=postgresql 74 | exclude: 75 | - rvm: 2.3 76 | gemfile: gemfiles/rails_6.gemfile 77 | - rvm: 2.4 78 | gemfile: gemfiles/rails_6.gemfile 79 | script: ./ci/000-prepare-database && bundle exec rake 80 | -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | appraise "rails-3" do 2 | gem "mysql2", "~> 0.3.10" 3 | gem "pg", "~> 0.11" 4 | gem "rails", "~> 3" 5 | gem "sqlite3", "~> 1.3.6" 6 | gem "test-unit", "~> 3.0" 7 | end 8 | 9 | appraise "rails-4" do 10 | gem "mysql2" 11 | gem "pg", "~> 0.15" 12 | gem "rails", "~> 4" 13 | gem "sqlite3", "~> 1.3.6" 14 | end 15 | 16 | appraise "rails-5" do 17 | gem "mysql2" 18 | gem "pg" 19 | gem "rails", "~> 5" 20 | gem "sqlite3" 21 | end 22 | 23 | appraise "rails-6" do 24 | gem "mysql2" 25 | gem "pg" 26 | gem "rails", "~> 6" 27 | gem "sqlite3" 28 | end 29 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 1.9.0 2 | ----- 3 | 4 | - Add CI test coverage for Rails 3-6 and Ruby 2.2-2.7 (#72) [@mberlanda] 5 | - Add CI mysql test coverage (#74 #75 #76 #77) [@mberlanda] 6 | - Restore rails < 5 mysql support (#78) [@mberlanda] 7 | - Add CI postgresql test coverage (#79) [@mberlanda] 8 | 9 | 10 | 1.8.2 11 | ----- 12 | 13 | - Replace 'type_cast_from_column' to support Rails 6.1 (#68) [@zjohl] 14 | 15 | 1.8.1 16 | ----- 17 | 18 | - Worker options ignore: false and update_duplicates: false cause an error when using postgresql_adapter (#60) [@torce] 19 | 20 | 1.8.0 21 | ----- 22 | 23 | - Abstract database-specific statements (#46) [@sobstel] 24 | - Allow to update duplicates on conflict in PostgreSQL (#40) [@sobstel] 25 | - Add CI on pull requests / merges (#38) [@mberlanda] 26 | 27 | 1.7.0 28 | ----- 29 | 30 | - Reduce requirements to allow rails 3 (#31) [Dmitry Ishkov] 31 | - Add backticks around "on duplicate key" columns (MySQL) (#33) [Mauro Berlanda] 32 | - PostgreSQL option to return primary keys (#32) [Peter Loomis] 33 | 34 | 1.6.0 35 | ----- 36 | 37 | - Support Mysql2 adapter (@varyform) 38 | - Add support for `update_duplicates` (@mstruve) 39 | - Add support for PostGIS, Mysql2Spatial (@knu) 40 | 41 | 1.5.0 42 | ----- 43 | 44 | - "Ignore" support for SQLite [@jfiander] 45 | - "Ignore" support for PostgreSQL [Mauro Berlanda] 46 | - add a callback for before_save [René Sprotte] 47 | 48 | 1.4.0 49 | ----- 50 | 51 | - better support for Rails 5 52 | - add an option for ignoring errors on insert 53 | 54 | 1.3.0 55 | ----- 56 | 57 | - Adds support for an "after save" callback on the worker. 58 | 59 | 1.2.0 60 | ----- 61 | 62 | - Fix Deprecation warning with ActiveRecord 5.0.0; 63 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Declare your gem's dependencies in bulk_insert.gemspec. 4 | # Bundler will treat runtime dependencies like base dependencies, and 5 | # development dependencies will be added by default to the :development group. 6 | gemspec 7 | 8 | # Declare any dependencies that are still in development here instead of in 9 | # your gemspec. These might include edge Rails or gems from your path or 10 | # Git. Remember to move these dependencies to your gemspec before releasing 11 | # your gem to rubygems.org. 12 | 13 | # To use a debugger 14 | # gem 'byebug', group: [:development, :test] 15 | 16 | gem "minitest" 17 | gem "rails", ">= 3.2.0" 18 | gem "sqlite3" 19 | -------------------------------------------------------------------------------- /MIT-LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 Jamis Buck 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BulkInsert 2 | 3 | A little ActiveRecord extension for helping to insert lots of rows in a 4 | single insert statement. 5 | 6 | ## Installation 7 | 8 | Add it to your Gemfile: 9 | 10 | ```ruby 11 | gem 'bulk_insert' 12 | ``` 13 | 14 | ## Usage 15 | 16 | BulkInsert adds a new class method to your ActiveRecord models: 17 | 18 | ```ruby 19 | class Book < ActiveRecord::Base 20 | end 21 | 22 | book_attrs = ... # some array of hashes, for instance 23 | Book.bulk_insert do |worker| 24 | book_attrs.each do |attrs| 25 | worker.add(attrs) 26 | end 27 | end 28 | ``` 29 | 30 | All of those `#add` calls will be accumulated into a single SQL insert 31 | statement, vastly improving the performance of multiple sequential 32 | inserts (think data imports and the like). 33 | 34 | If you don't like using a block API, you can also simply pass an array 35 | of rows to be inserted: 36 | 37 | ```ruby 38 | book_attrs = ... # some array of hashes, for instance 39 | Book.bulk_insert values: book_attrs 40 | ``` 41 | 42 | By default, the columns to be inserted will be all columns in the table, 43 | minus the `id` column, but if you want, you can explicitly enumerate 44 | the columns: 45 | 46 | ```ruby 47 | Book.bulk_insert(:title, :author) do |worker| 48 | # specify a row as an array of values... 49 | worker.add ["Eye of the World", "Robert Jordan"] 50 | 51 | # or as a hash 52 | worker.add title: "Lord of Light", author: "Roger Zelazny" 53 | end 54 | ``` 55 | 56 | It will automatically set `created_at`/`updated_at` columns to the current 57 | date, as well. 58 | 59 | ```ruby 60 | Book.bulk_insert(:title, :author, :created_at, :updated_at) do |worker| 61 | # specify created_at/updated_at explicitly... 62 | worker.add ["The Chosen", "Chaim Potok", Time.now, Time.now] 63 | 64 | # or let BulkInsert set them by default... 65 | worker.add ["Hello Ruby", "Linda Liukas"] 66 | end 67 | ``` 68 | 69 | Similarly, if a value is omitted, BulkInsert will use whatever default 70 | value is defined for that column in the database: 71 | 72 | ```ruby 73 | # create_table :books do |t| 74 | # ... 75 | # t.string "medium", default: "paper" 76 | # ... 77 | # end 78 | 79 | Book.bulk_insert(:title, :author, :medium) do |worker| 80 | worker.add title: "Ender's Game", author: "Orson Scott Card" 81 | end 82 | 83 | Book.first.medium #-> "paper" 84 | ``` 85 | 86 | By default, the batch is always saved when the block finishes, but you 87 | can explicitly save inside the block whenever you want, by calling 88 | `#save!` on the worker: 89 | 90 | ```ruby 91 | Book.bulk_insert do |worker| 92 | worker.add(...) 93 | worker.add(...) 94 | 95 | worker.save! 96 | 97 | worker.add(...) 98 | #... 99 | end 100 | ``` 101 | 102 | That will save the batch as it has been defined to that point, and then 103 | empty the batch so that you can add more rows to it if you want. Note 104 | that all records saved together will have the same created_at/updated_at 105 | timestamp (unless one was explicitly set). 106 | 107 | ### Batch Set Size 108 | 109 | By default, the size of the insert is limited to 500 rows at a time. 110 | This is called the _set size_. If you add another row that causes the 111 | set to exceed the set size, the insert statement is automatically built 112 | and executed, and the batch is reset. 113 | 114 | If you want a larger (or smaller) set size, you can specify it in 115 | two ways: 116 | 117 | ```ruby 118 | # specify set_size when initializing the bulk insert... 119 | Book.bulk_insert(set_size: 100) do |worker| 120 | # ... 121 | end 122 | 123 | # specify it on the worker directly... 124 | Book.bulk_insert do |worker| 125 | worker.set_size = 100 126 | # ... 127 | end 128 | ``` 129 | 130 | ### Insert Ignore 131 | 132 | By default, when an insert fails the whole batch of inserts fail. The 133 | _ignore_ option ignores the inserts that would have failed (because of 134 | duplicate keys or a null in column with a not null constraint) and 135 | inserts the rest of the batch. 136 | 137 | This is not the default because no errors are raised for the bad 138 | inserts in the batch. 139 | 140 | ```ruby 141 | destination_columns = [:title, :author] 142 | 143 | # Ignore bad inserts in the batch 144 | Book.bulk_insert(*destination_columns, ignore: true) do |worker| 145 | worker.add(...) 146 | worker.add(...) 147 | # ... 148 | end 149 | ``` 150 | 151 | ### Update Duplicates (MySQL, PostgreSQL) 152 | 153 | If you don't want to ignore duplicate rows but instead want to update them 154 | then you can use the _update_duplicates_ option. Set this option to true 155 | (MySQL) or list unique column names (PostgreSQL) and when a duplicate row 156 | is found the row will be updated with your new values. 157 | Default value for this option is false. 158 | 159 | ```ruby 160 | destination_columns = [:title, :author] 161 | 162 | # Update duplicate rows (MySQL) 163 | Book.bulk_insert(*destination_columns, update_duplicates: true) do |worker| 164 | worker.add(...) 165 | worker.add(...) 166 | # ... 167 | end 168 | 169 | # Update duplicate rows (PostgreSQL) 170 | Book.bulk_insert(*destination_columns, update_duplicates: %w[title]) do |worker| 171 | worker.add(...) 172 | # ... 173 | end 174 | ``` 175 | 176 | ### Return Primary Keys (PostgreSQL, PostGIS) 177 | 178 | If you want the worker to store primary keys of inserted records, then you can 179 | use the _return_primary_keys_ option. The worker will store a `result_sets` 180 | array of `ActiveRecord::Result` objects. Each `ActiveRecord::Result` object 181 | will contain the primary keys of a batch of inserted records. 182 | 183 | ```ruby 184 | worker = Book.bulk_insert(*destination_columns, return_primary_keys: true) do 185 | |worker| 186 | worker.add(...) 187 | worker.add(...) 188 | # ... 189 | end 190 | 191 | worker.result_sets 192 | ``` 193 | 194 | ## Ruby and Rails Versions Supported 195 | 196 | > :warning: The scope of this gem may be somehow covered natively by the `.insert_all` API 197 | > introduced by [Rails 6](https://apidock.com/rails/v6.0.0/ActiveRecord/Persistence/ClassMethods/insert_all). 198 | > This gem represents the state of art for rails version < 6 and it is still open to 199 | > further developments for more recent versions. 200 | 201 | The current CI prevents regressions on the following versions: 202 | 203 | ruby / rails | `~>3` | `~>4` | `~>5` | `~>6` 204 | :-----------:|-------|-------|-------|------ 205 | 2.2 | yes | yes | no | no 206 | 2.3 | yes | yes | yes | no 207 | 2.4 | no | yes | yes | no 208 | 2.5 | no | no | yes | yes 209 | 2.6 | no | no | yes | yes 210 | 2.7 | no | no | yes | yes 211 | 212 | The adapters covered in the CI are: 213 | * sqlite 214 | * mysql 215 | * postgresql 216 | 217 | 218 | ## License 219 | 220 | BulkInsert is released under the MIT license (see MIT-LICENSE) by 221 | Jamis Buck (jamis@jamisbuck.org). 222 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | begin 2 | require 'bundler/setup' 3 | rescue LoadError 4 | puts 'You must `gem install bundler` and `bundle install` to run rake tasks' 5 | end 6 | 7 | require 'rdoc/task' 8 | 9 | RDoc::Task.new(:rdoc) do |rdoc| 10 | rdoc.rdoc_dir = 'rdoc' 11 | rdoc.title = 'BulkInsert' 12 | rdoc.options << '--line-numbers' 13 | rdoc.rdoc_files.include('README.rdoc') 14 | rdoc.rdoc_files.include('lib/**/*.rb') 15 | end 16 | 17 | Bundler::GemHelper.install_tasks 18 | 19 | require 'rake/testtask' 20 | 21 | Rake::TestTask.new(:test) do |t| 22 | t.libs << 'lib' 23 | t.libs << 'test' 24 | t.pattern = 'test/**/*_test.rb' 25 | t.verbose = false 26 | end 27 | 28 | 29 | task default: :test 30 | -------------------------------------------------------------------------------- /bulk_insert.gemspec: -------------------------------------------------------------------------------- 1 | $:.push File.expand_path("../lib", __FILE__) 2 | 3 | # Maintain your gem's version: 4 | require "bulk_insert/version" 5 | 6 | # Describe your gem and declare its dependencies: 7 | Gem::Specification.new do |s| 8 | s.name = "bulk_insert" 9 | s.version = BulkInsert::VERSION 10 | s.authors = ["Jamis Buck", "Mauro Berlanda"] 11 | s.email = ["jamis@jamisbuck.org", "mauro.berlanda@gmail.com"] 12 | s.homepage = "http://github.com/jamis/bulk_insert" 13 | s.summary = "An helper for doing batch (single-statement) inserts in ActiveRecord" 14 | s.description = "Faster inserts! Insert N records in a single statement." 15 | s.license = "MIT" 16 | 17 | s.files = Dir["{app,config,db,lib}/**/*", "MIT-LICENSE", "Rakefile", "README.md"] 18 | s.test_files = Dir["test/**/*"] 19 | 20 | s.add_dependency "activerecord", ">= 3.2.0" 21 | 22 | s.add_development_dependency "appraisal" 23 | end 24 | -------------------------------------------------------------------------------- /ci/000-prepare-database: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | export DB_ADAPTER=${DB_ADAPTER:-mysql} 5 | 6 | case $DB_ADAPTER in 7 | mysql) 8 | mysql -e 'CREATE DATABASE bulk_insert_test;' 9 | ;; 10 | postgresql) 11 | psql -c 'create database bulk_insert_test;' -U postgres 12 | ;; 13 | esac 14 | 15 | cd test/dummy 16 | bundle exec rake db:test:load 17 | cd - 18 | -------------------------------------------------------------------------------- /gemfiles/rails_3.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "minitest" 6 | gem "rails", "~> 3" 7 | gem "sqlite3", "~> 1.3.6" 8 | gem "mysql2", "~> 0.3.10" 9 | gem "pg", "~> 0.11" 10 | gem "test-unit", "~> 3.0" 11 | 12 | gemspec path: "../" 13 | -------------------------------------------------------------------------------- /gemfiles/rails_4.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "minitest" 6 | gem "rails", "~> 4" 7 | gem "sqlite3", "~> 1.3.6" 8 | gem "mysql2" 9 | gem "pg", "~> 0.15" 10 | 11 | gemspec path: "../" 12 | -------------------------------------------------------------------------------- /gemfiles/rails_5.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "minitest" 6 | gem "rails", "~> 5" 7 | gem "sqlite3" 8 | gem "mysql2" 9 | gem "pg" 10 | 11 | gemspec path: "../" 12 | -------------------------------------------------------------------------------- /gemfiles/rails_6.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "minitest" 6 | gem "rails", "~> 6" 7 | gem "sqlite3" 8 | gem "mysql2" 9 | gem "pg" 10 | 11 | gemspec path: "../" 12 | -------------------------------------------------------------------------------- /lib/bulk_insert.rb: -------------------------------------------------------------------------------- 1 | require 'bulk_insert/worker' 2 | 3 | module BulkInsert 4 | extend ActiveSupport::Concern 5 | 6 | module ClassMethods 7 | def bulk_insert(*columns, values: nil, set_size:500, ignore: false, update_duplicates: false, return_primary_keys: false) 8 | columns = default_bulk_columns if columns.empty? 9 | worker = BulkInsert::Worker.new(connection, table_name, primary_key, columns, set_size, ignore, update_duplicates, return_primary_keys) 10 | 11 | if values.present? 12 | transaction do 13 | worker.add_all(values) 14 | worker.save! 15 | end 16 | nil 17 | elsif block_given? 18 | transaction do 19 | yield worker 20 | worker.save! 21 | end 22 | nil 23 | else 24 | worker 25 | end 26 | end 27 | 28 | # helper method for preparing the columns before a call to :bulk_insert 29 | def default_bulk_columns 30 | self.column_names - %w(id) 31 | end 32 | 33 | end 34 | end 35 | 36 | ActiveSupport.on_load(:active_record) do 37 | send(:include, BulkInsert) 38 | end 39 | -------------------------------------------------------------------------------- /lib/bulk_insert/statement_adapters.rb: -------------------------------------------------------------------------------- 1 | require_relative 'statement_adapters/generic_adapter' 2 | require_relative 'statement_adapters/mysql_adapter' 3 | require_relative 'statement_adapters/postgresql_adapter' 4 | require_relative 'statement_adapters/sqlite_adapter' 5 | 6 | module BulkInsert 7 | module StatementAdapters 8 | def adapter_for(connection) 9 | case connection.adapter_name 10 | when /^mysql/i 11 | MySQLAdapter.new 12 | when /\APost(?:greSQL|GIS)/i 13 | PostgreSQLAdapter.new 14 | when /\ASQLite/i 15 | SQLiteAdapter.new 16 | else 17 | GenericAdapter.new 18 | end 19 | end 20 | module_function :adapter_for 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/bulk_insert/statement_adapters/base_adapter.rb: -------------------------------------------------------------------------------- 1 | module BulkInsert 2 | module StatementAdapters 3 | class BaseAdapter 4 | def initialize 5 | raise "You cannot initialize base adapter" if self.class == BaseAdapter 6 | end 7 | 8 | def insert_ignore_statement 9 | raise "Not implemented" 10 | end 11 | 12 | def on_conflict_statement(_columns, _ignore, _update_duplicates) 13 | raise "Not implemented" 14 | end 15 | 16 | def primary_key_return_statement(_primary_key) 17 | raise "Not implemented" 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/bulk_insert/statement_adapters/generic_adapter.rb: -------------------------------------------------------------------------------- 1 | require_relative 'base_adapter' 2 | 3 | module BulkInsert 4 | module StatementAdapters 5 | class GenericAdapter < BaseAdapter 6 | def insert_ignore_statement 7 | '' 8 | end 9 | 10 | def on_conflict_statement(_columns, _ignore, _update_duplicates) 11 | '' 12 | end 13 | 14 | def primary_key_return_statement(_primary_key) 15 | '' 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/bulk_insert/statement_adapters/mysql_adapter.rb: -------------------------------------------------------------------------------- 1 | require_relative 'base_adapter' 2 | 3 | module BulkInsert 4 | module StatementAdapters 5 | class MySQLAdapter < BaseAdapter 6 | def insert_ignore_statement 7 | 'IGNORE' 8 | end 9 | 10 | def on_conflict_statement(columns, _ignore, update_duplicates) 11 | return '' unless update_duplicates 12 | 13 | update_values = columns.map do |column| 14 | "`#{column.name}`=VALUES(`#{column.name}`)" 15 | end.join(', ') 16 | ' ON DUPLICATE KEY UPDATE ' + update_values 17 | end 18 | 19 | def primary_key_return_statement(_primary_key) 20 | '' 21 | end 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/bulk_insert/statement_adapters/postgresql_adapter.rb: -------------------------------------------------------------------------------- 1 | require_relative 'base_adapter' 2 | 3 | module BulkInsert 4 | module StatementAdapters 5 | class PostgreSQLAdapter < BaseAdapter 6 | def insert_ignore_statement 7 | '' 8 | end 9 | 10 | def on_conflict_statement(columns, ignore, update_duplicates) 11 | if ignore 12 | ' ON CONFLICT DO NOTHING' 13 | elsif update_duplicates 14 | update_values = columns.map do |column| 15 | "#{column.name}=EXCLUDED.#{column.name}" 16 | end.join(', ') 17 | ' ON CONFLICT(' + update_duplicates.join(', ') + ') DO UPDATE SET ' + update_values 18 | else 19 | '' 20 | end 21 | end 22 | 23 | def primary_key_return_statement(primary_key) 24 | " RETURNING #{primary_key}" 25 | end 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/bulk_insert/statement_adapters/sqlite_adapter.rb: -------------------------------------------------------------------------------- 1 | require_relative 'base_adapter' 2 | 3 | module BulkInsert 4 | module StatementAdapters 5 | class SQLiteAdapter < BaseAdapter 6 | def insert_ignore_statement 7 | 'OR IGNORE' 8 | end 9 | 10 | def on_conflict_statement(_columns, _ignore, _update_duplicates) 11 | '' 12 | end 13 | 14 | def primary_key_return_statement(_primary_key) 15 | '' 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/bulk_insert/version.rb: -------------------------------------------------------------------------------- 1 | module BulkInsert 2 | MAJOR = 1 3 | MINOR = 9 4 | TINY = 0 5 | 6 | VERSION = [MAJOR, MINOR, TINY].join(".") 7 | end 8 | -------------------------------------------------------------------------------- /lib/bulk_insert/worker.rb: -------------------------------------------------------------------------------- 1 | require_relative 'statement_adapters' 2 | 3 | module BulkInsert 4 | class Worker 5 | attr_reader :connection 6 | attr_accessor :set_size 7 | attr_accessor :before_save_callback 8 | attr_accessor :after_save_callback 9 | attr_accessor :adapter_name 10 | attr_reader :ignore, :update_duplicates, :result_sets 11 | 12 | def initialize(connection, table_name, primary_key, column_names, set_size=500, ignore=false, update_duplicates=false, return_primary_keys=false) 13 | @statement_adapter = StatementAdapters.adapter_for(connection) 14 | 15 | @connection = connection 16 | @set_size = set_size 17 | 18 | @adapter_name = connection.adapter_name 19 | # INSERT IGNORE only fails inserts with duplicate keys or unallowed nulls not the whole set of inserts 20 | @ignore = ignore 21 | @update_duplicates = update_duplicates 22 | @return_primary_keys = return_primary_keys 23 | 24 | columns = connection.columns(table_name) 25 | column_map = columns.inject({}) { |h, c| h.update(c.name => c) } 26 | 27 | @primary_key = primary_key 28 | @columns = column_names.map { |name| column_map[name.to_s] } 29 | @table_name = connection.quote_table_name(table_name) 30 | @column_names = column_names.map { |name| connection.quote_column_name(name) }.join(",") 31 | 32 | @before_save_callback = nil 33 | @after_save_callback = nil 34 | 35 | @result_sets = [] 36 | @set = [] 37 | end 38 | 39 | def pending? 40 | @set.any? 41 | end 42 | 43 | def pending_count 44 | @set.count 45 | end 46 | 47 | def add(values) 48 | save! if @set.length >= set_size 49 | 50 | values = values.with_indifferent_access if values.is_a?(Hash) 51 | mapped = @columns.map.with_index do |column, index| 52 | value_exists = values.is_a?(Hash) ? values.key?(column.name) : (index < values.length) 53 | if !value_exists 54 | if column.default.present? 55 | column.default 56 | elsif column.name == "created_at" || column.name == "updated_at" 57 | :__timestamp_placeholder 58 | else 59 | nil 60 | end 61 | else 62 | values.is_a?(Hash) ? values[column.name] : values[index] 63 | end 64 | end 65 | 66 | @set.push(mapped) 67 | self 68 | end 69 | 70 | def add_all(rows) 71 | rows.each { |row| add(row) } 72 | self 73 | end 74 | 75 | def before_save(&block) 76 | @before_save_callback = block 77 | end 78 | 79 | def after_save(&block) 80 | @after_save_callback = block 81 | end 82 | 83 | def save! 84 | if pending? 85 | @before_save_callback.(@set) if @before_save_callback 86 | execute_query 87 | @after_save_callback.() if @after_save_callback 88 | @set.clear 89 | end 90 | 91 | self 92 | end 93 | 94 | def execute_query 95 | if query = compose_insert_query 96 | 97 | # Return primary key support broke mysql compatibility 98 | # with rails < 5 mysql adapter. (see issue #41) 99 | if ActiveRecord::VERSION::STRING < "5.0.0" && @statement_adapter.is_a?(StatementAdapters::MySQLAdapter) 100 | # raise an exception for unsupported return_primary_keys 101 | raise ArgumentError.new("BulkInsert does not support @return_primary_keys for mysql and rails < 5") if @return_primary_keys 102 | 103 | # restore v1.6 query execution 104 | @connection.execute(query) 105 | else 106 | result_set = @connection.exec_query(query) 107 | @result_sets.push(result_set) if @return_primary_keys 108 | end 109 | end 110 | end 111 | 112 | def compose_insert_query 113 | sql = insert_sql_statement 114 | @now = Time.now 115 | rows = [] 116 | 117 | @set.each do |row| 118 | values = [] 119 | @columns.zip(row) do |column, value| 120 | value = @now if value == :__timestamp_placeholder 121 | 122 | if ActiveRecord::VERSION::STRING >= "5.0.0" 123 | if column 124 | type = @connection.lookup_cast_type_from_column(column) 125 | value = type.serialize(value) 126 | end 127 | values << @connection.quote(value) 128 | else 129 | values << @connection.quote(value, column) 130 | end 131 | end 132 | rows << "(#{values.join(',')})" 133 | end 134 | 135 | if !rows.empty? 136 | sql << rows.join(",") 137 | sql << @statement_adapter.on_conflict_statement(@columns, ignore, update_duplicates) 138 | sql << @statement_adapter.primary_key_return_statement(@primary_key) if @return_primary_keys 139 | sql 140 | else 141 | false 142 | end 143 | end 144 | 145 | def insert_sql_statement 146 | insert_ignore = @ignore ? @statement_adapter.insert_ignore_statement : '' 147 | "INSERT #{insert_ignore} INTO #{@table_name} (#{@column_names}) VALUES " 148 | end 149 | end 150 | end 151 | -------------------------------------------------------------------------------- /test/bulk_insert/worker_test.rb: -------------------------------------------------------------------------------- 1 | require 'minitest/mock' 2 | require 'test_helper' 3 | require 'connection_mocks' 4 | 5 | class BulkInsertWorkerTest < ActiveSupport::TestCase 6 | include ConnectionMocks 7 | 8 | setup do 9 | @insert = BulkInsert::Worker.new( 10 | Testing.connection, 11 | Testing.table_name, 12 | 'id', 13 | %w(greeting age happy created_at updated_at color)) 14 | @now = Time.now.utc 15 | end 16 | 17 | test "empty insert is not pending" do 18 | assert_equal false, @insert.pending? 19 | end 20 | 21 | test "pending_count should describe size of pending set" do 22 | assert_equal 0, @insert.pending_count 23 | @insert.add ["Hello", 15, true, @now, @now] 24 | assert_equal 1, @insert.pending_count 25 | end 26 | 27 | test "default set size" do 28 | assert_equal 500, @insert.set_size 29 | end 30 | 31 | test "adding row to insert makes insert pending" do 32 | @insert.add ["Hello", 15, true, @now, @now] 33 | assert_equal true, @insert.pending? 34 | end 35 | 36 | test "add should default timestamp columns to current time" do 37 | now = Time.now 38 | 39 | @insert.add ["Hello", 15, true] 40 | @insert.save! 41 | 42 | record = Testing.first 43 | assert_operator record.created_at.to_i, :>=, now.to_i 44 | assert_operator record.updated_at.to_i, :>=, now.to_i 45 | end 46 | 47 | test "default timestamp columns should be equivalent for the entire batch" do 48 | @insert.add ["Hello", 15, true] 49 | @insert.add ["Howdy", 20, false] 50 | @insert.save! 51 | 52 | first, second = Testing.all 53 | assert_equal first.created_at.to_f, second.created_at.to_f 54 | assert_equal first.created_at.to_f, first.updated_at.to_f 55 | end 56 | 57 | test "add should use database default values when present" do 58 | @insert.add greeting: "Hello", age: 20, happy: false 59 | @insert.save! 60 | 61 | record = Testing.first 62 | assert_equal record.color, "chartreuse" 63 | end 64 | 65 | test "explicit nil should override defaults" do 66 | @insert.add greeting: "Hello", age: 20, happy: false, color: nil 67 | @insert.save! 68 | 69 | record = Testing.first 70 | assert_nil record.color 71 | end 72 | 73 | test "add should allow values given as Hash" do 74 | @insert.add greeting: "Yo", age: 20, happy: false, created_at: @now, updated_at: @now 75 | @insert.save! 76 | 77 | record = Testing.first 78 | assert_not_nil record 79 | assert_equal "Yo", record.greeting 80 | assert_equal 20, record.age 81 | assert_equal false, record.happy? 82 | end 83 | 84 | test "add should save automatically when overflowing set size" do 85 | @insert.set_size = 1 86 | @insert.add ["Hello", 15, true, @now, @now] 87 | @insert.add ["Yo", 20, false, @now, @now] 88 | assert_equal 1, Testing.count 89 | assert_equal "Hello", Testing.first.greeting 90 | end 91 | 92 | test "add_all should append all items to the set" do 93 | @insert.add_all [ 94 | [ "Hello", 15, true ], 95 | { greeting: "Hi", age: 55, happy: true } 96 | ] 97 | assert_equal 2, @insert.pending_count 98 | end 99 | 100 | test "save! makes insert not pending" do 101 | @insert.add ["Hello", 15, true, @now, @now] 102 | @insert.save! 103 | assert_equal false, @insert.pending? 104 | end 105 | 106 | test "save! when not pending should do nothing" do 107 | assert_no_difference 'Testing.count' do 108 | @insert.save! 109 | end 110 | end 111 | 112 | test "save! inserts pending records" do 113 | @insert.add ["Yo", 15, false, @now, @now] 114 | @insert.add ["Hello", 25, true, @now, @now] 115 | @insert.save! 116 | 117 | yo = Testing.where(greeting: 'Yo').first 118 | hello = Testing.where(greeting: 'Hello').first 119 | 120 | assert_not_nil yo 121 | assert_equal 15, yo.age 122 | assert_equal false, yo.happy? 123 | 124 | assert_not_nil hello 125 | assert_equal 25, hello.age 126 | assert_equal true, hello.happy? 127 | end 128 | 129 | test "save! does not add to result sets when not returning primary keys" do 130 | @insert.add greeting: "first" 131 | @insert.add greeting: "second" 132 | @insert.save! 133 | 134 | assert_equal 0, @insert.result_sets.count 135 | end 136 | 137 | 138 | test "save! adds to result sets when returning primary keys" do 139 | worker = BulkInsert::Worker.new( 140 | Testing.connection, 141 | Testing.table_name, 142 | 'id', 143 | %w(greeting age happy created_at updated_at color), 144 | 500, 145 | false, 146 | false, 147 | true 148 | ) 149 | 150 | # return_primary_keys is not supported for mysql and rails < 5 151 | # skip is not supported in the minitest version used for testing rails 3 152 | return if ActiveRecord::VERSION::STRING < "5.0.0" && worker.adapter_name =~ /^mysql/i 153 | 154 | assert_no_difference -> { worker.result_sets.count } do 155 | worker.save! 156 | end 157 | 158 | worker.add greeting: "first" 159 | worker.add greeting: "second" 160 | worker.save! 161 | assert_equal 1, worker.result_sets.count 162 | 163 | worker.add greeting: "third" 164 | worker.add greeting: "fourth" 165 | worker.save! 166 | assert_equal 2, worker.result_sets.count 167 | end 168 | 169 | test "initialized with empty result sets array" do 170 | new_worker = BulkInsert::Worker.new( 171 | Testing.connection, 172 | Testing.table_name, 173 | 'id', 174 | %w(greeting age happy created_at updated_at color) 175 | ) 176 | assert_instance_of(Array, new_worker.result_sets) 177 | assert_empty new_worker.result_sets 178 | end 179 | 180 | test "save! calls the after_save handler" do 181 | x = 41 182 | 183 | @insert.after_save do 184 | x += 1 185 | end 186 | 187 | @insert.add ["Yo", 15, false, @now, @now] 188 | @insert.add ["Hello", 25, true, @now, @now] 189 | @insert.save! 190 | 191 | assert_equal 42, x 192 | end 193 | 194 | test "after_save stores a block as a proc" do 195 | @insert.after_save do 196 | "hello" 197 | end 198 | 199 | assert_equal "hello", @insert.after_save_callback.() 200 | end 201 | 202 | test "after_save_callback can be set as a proc" do 203 | @insert.after_save_callback = -> do 204 | "hello" 205 | end 206 | 207 | assert_equal "hello", @insert.after_save_callback.() 208 | end 209 | 210 | test "save! calls the before_save handler" do 211 | x = 41 212 | 213 | @insert.before_save do 214 | x += 1 215 | end 216 | 217 | @insert.add ["Yo", 15, false, @now, @now] 218 | @insert.add ["Hello", 25, true, @now, @now] 219 | @insert.save! 220 | 221 | assert_equal 42, x 222 | end 223 | 224 | test "before_save stores a block as a proc" do 225 | @insert.before_save do 226 | "hello" 227 | end 228 | 229 | assert_equal "hello", @insert.before_save_callback.() 230 | end 231 | 232 | test "before_save_callback can be set as a proc" do 233 | @insert.before_save_callback = -> do 234 | "hello" 235 | end 236 | 237 | assert_equal "hello", @insert.before_save_callback.() 238 | end 239 | 240 | test "before_save can manipulate the set" do 241 | @insert.before_save do |set| 242 | set.reject!{|row| row[0] == "Yo"} 243 | end 244 | 245 | @insert.add ["Yo", 15, false, @now, @now] 246 | @insert.add ["Hello", 25, true, @now, @now] 247 | @insert.save! 248 | 249 | yo = Testing.where(greeting: 'Yo').first 250 | hello = Testing.where(greeting: 'Hello').first 251 | 252 | assert_nil yo 253 | assert_not_nil hello 254 | end 255 | 256 | test "save! doesn't blow up if before_save emptying the set" do 257 | @insert.before_save do |set| 258 | set.clear 259 | end 260 | 261 | @insert.add ["Yo", 15, false, @now, @now] 262 | @insert.add ["Hello", 25, true, @now, @now] 263 | @insert.save! 264 | 265 | yo = Testing.where(greeting: 'Yo').first 266 | hello = Testing.where(greeting: 'Hello').first 267 | 268 | assert_nil yo 269 | assert_nil hello 270 | end 271 | 272 | test "adapter dependent SQLite methods" do 273 | connection = Testing.connection 274 | stub_connection_if_needed(connection, 'SQLite') do 275 | sqlite_worker = BulkInsert::Worker.new( 276 | connection, 277 | Testing.table_name, 278 | 'id', 279 | %w(greeting age happy created_at updated_at color), 280 | 500 # batch size 281 | ) 282 | 283 | assert_equal sqlite_worker.adapter_name, 'SQLite' 284 | assert_equal sqlite_worker.insert_sql_statement, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES " 285 | 286 | sqlite_worker.add ["Yo", 15, false, nil, nil] 287 | assert_equal sqlite_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')" 288 | end 289 | end 290 | 291 | test "adapter dependent MySQL methods" do 292 | connection = Testing.connection 293 | stub_connection_if_needed(connection, 'mysql') do 294 | mysql_worker = BulkInsert::Worker.new( 295 | connection, 296 | Testing.table_name, 297 | 'id', 298 | %w(greeting age happy created_at updated_at color), 299 | 500, # batch size 300 | true # ignore 301 | ) 302 | 303 | assert_equal mysql_worker.adapter_name, 'mysql' 304 | assert_equal (mysql_worker.adapter_name == 'mysql'), true 305 | assert_equal mysql_worker.ignore, true 306 | assert_equal ((mysql_worker.adapter_name == 'mysql') & mysql_worker.ignore), true 307 | 308 | mysql_worker.add ["Yo", 15, false, nil, nil] 309 | 310 | assert_statement_adapter mysql_worker, 'BulkInsert::StatementAdapters::MySQLAdapter' 311 | assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO `testings` (`greeting`,`age`,`happy`,`created_at`,`updated_at`,`color`) VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse')" 312 | end 313 | end 314 | 315 | test "adapter dependent mysql methods work for mysql2" do 316 | connection = Testing.connection 317 | stub_connection_if_needed(connection, 'mysql2') do 318 | mysql_worker = BulkInsert::Worker.new( 319 | connection, 320 | Testing.table_name, 321 | 'id', 322 | %w(greeting age happy created_at updated_at color), 323 | 500, # batch size 324 | true, # ignore 325 | true) # update_duplicates 326 | 327 | assert_equal mysql_worker.adapter_name, 'mysql2' 328 | assert mysql_worker.ignore 329 | 330 | mysql_worker.add ["Yo", 15, false, nil, nil] 331 | 332 | assert_statement_adapter mysql_worker, 'BulkInsert::StatementAdapters::MySQLAdapter' 333 | assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO `testings` (`greeting`,`age`,`happy`,`created_at`,`updated_at`,`color`) VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)" 334 | end 335 | end 336 | 337 | test "adapter dependent Mysql2Spatial methods" do 338 | connection = Testing.connection 339 | stub_connection_if_needed(connection, 'mysql2spatial') do 340 | mysql_worker = BulkInsert::Worker.new( 341 | connection, 342 | Testing.table_name, 343 | 'id', 344 | %w(greeting age happy created_at updated_at color), 345 | 500, # batch size 346 | true) # ignore 347 | 348 | assert_equal mysql_worker.adapter_name, 'mysql2spatial' 349 | 350 | mysql_worker.add ["Yo", 15, false, nil, nil] 351 | 352 | assert_statement_adapter mysql_worker, 'BulkInsert::StatementAdapters::MySQLAdapter' 353 | assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO `testings` (`greeting`,`age`,`happy`,`created_at`,`updated_at`,`color`) VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse')" 354 | end 355 | end 356 | 357 | test "adapter dependent postgresql methods" do 358 | connection = Testing.connection 359 | stub_connection_if_needed(connection, 'PostgreSQL') do 360 | pgsql_worker = BulkInsert::Worker.new( 361 | connection, 362 | Testing.table_name, 363 | 'id', 364 | %w(greeting age happy created_at updated_at color), 365 | 500, # batch size 366 | true, # ignore 367 | false, # update duplicates 368 | true # return primary keys 369 | ) 370 | 371 | pgsql_worker.add ["Yo", 15, false, nil, nil] 372 | 373 | assert_statement_adapter pgsql_worker, 'BulkInsert::StatementAdapters::PostgreSQLAdapter' 374 | 375 | if ActiveRecord::VERSION::STRING >= "5.0.0" 376 | assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id" 377 | else 378 | assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,'f',NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id" 379 | end 380 | end 381 | end 382 | 383 | test "adapter dependent postgresql methods (no ignore, no update_duplicates)" do 384 | connection = Testing.connection 385 | stub_connection_if_needed(connection, 'PostgreSQL') do 386 | pgsql_worker = BulkInsert::Worker.new( 387 | connection, 388 | Testing.table_name, 389 | 'id', 390 | %w(greeting age happy created_at updated_at color), 391 | 500, # batch size 392 | false, # ignore 393 | false, # update duplicates 394 | true # return primary keys 395 | ) 396 | 397 | pgsql_worker.add ["Yo", 15, false, nil, nil] 398 | 399 | assert_statement_adapter pgsql_worker, 'BulkInsert::StatementAdapters::PostgreSQLAdapter' 400 | 401 | if ActiveRecord::VERSION::STRING >= "5.0.0" 402 | assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') RETURNING id" 403 | else 404 | assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,'f',NULL,NULL,'chartreuse') RETURNING id" 405 | end 406 | end 407 | end 408 | 409 | test "adapter dependent postgresql methods (with update_duplicates)" do 410 | connection = Testing.connection 411 | stub_connection_if_needed(connection, 'PostgreSQL') do 412 | pgsql_worker = BulkInsert::Worker.new( 413 | connection, 414 | Testing.table_name, 415 | 'id', 416 | %w(greeting age happy created_at updated_at color), 417 | 500, # batch size 418 | false, # ignore 419 | %w(greeting age happy), # update duplicates 420 | true # return primary keys 421 | ) 422 | pgsql_worker.add ["Yo", 15, false, nil, nil] 423 | 424 | assert_statement_adapter pgsql_worker, 'BulkInsert::StatementAdapters::PostgreSQLAdapter' 425 | 426 | if ActiveRecord::VERSION::STRING >= "5.0.0" 427 | assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON CONFLICT(greeting, age, happy) DO UPDATE SET greeting=EXCLUDED.greeting, age=EXCLUDED.age, happy=EXCLUDED.happy, created_at=EXCLUDED.created_at, updated_at=EXCLUDED.updated_at, color=EXCLUDED.color RETURNING id" 428 | else 429 | assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,'f',NULL,NULL,'chartreuse') ON CONFLICT(greeting, age, happy) DO UPDATE SET greeting=EXCLUDED.greeting, age=EXCLUDED.age, happy=EXCLUDED.happy, created_at=EXCLUDED.created_at, updated_at=EXCLUDED.updated_at, color=EXCLUDED.color RETURNING id" 430 | end 431 | end 432 | end 433 | 434 | test "adapter dependent PostGIS methods" do 435 | connection = Testing.connection 436 | stub_connection_if_needed(connection, 'postgis') do 437 | pgsql_worker = BulkInsert::Worker.new( 438 | connection, 439 | Testing.table_name, 440 | 'id', 441 | %w(greeting age happy created_at updated_at color), 442 | 500, # batch size 443 | true, # ignore 444 | false, # update duplicates 445 | true # return primary keys 446 | ) 447 | pgsql_worker.add ["Yo", 15, false, nil, nil] 448 | 449 | assert_statement_adapter pgsql_worker, 'BulkInsert::StatementAdapters::PostgreSQLAdapter' 450 | 451 | if ActiveRecord::VERSION::STRING >= "5.0.0" 452 | assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id" 453 | else 454 | assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,'f',NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id" 455 | end 456 | end 457 | end 458 | 459 | test "adapter dependent sqlite3 methods (with lowercase adapter name)" do 460 | connection = Testing.connection 461 | stub_connection_if_needed(connection, 'sqlite3') do 462 | sqlite_worker = BulkInsert::Worker.new( 463 | Testing.connection, 464 | Testing.table_name, 465 | 'id', 466 | %w(greeting age happy created_at updated_at color), 467 | 500, # batch size 468 | true) # ignore 469 | sqlite_worker.adapter_name = 'sqlite3' 470 | sqlite_worker.add ["Yo", 15, false, nil, nil] 471 | 472 | assert_statement_adapter sqlite_worker, 'BulkInsert::StatementAdapters::SQLiteAdapter' 473 | assert_equal sqlite_worker.compose_insert_query, "INSERT OR IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')" 474 | end 475 | end 476 | 477 | test "adapter dependent sqlite3 methods (with stylecase adapter name)" do 478 | connection = Testing.connection 479 | stub_connection_if_needed(connection, 'SQLite') do 480 | sqlite_worker = BulkInsert::Worker.new( 481 | connection, 482 | Testing.table_name, 483 | 'id', 484 | %w(greeting age happy created_at updated_at color), 485 | 500, # batch size 486 | true) # ignore 487 | sqlite_worker.adapter_name = 'SQLite' 488 | sqlite_worker.add ["Yo", 15, false, nil, nil] 489 | 490 | assert_statement_adapter sqlite_worker, 'BulkInsert::StatementAdapters::SQLiteAdapter' 491 | assert_equal sqlite_worker.compose_insert_query, "INSERT OR IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')" 492 | end 493 | end 494 | 495 | test "mysql adapter can update duplicates" do 496 | connection = Testing.connection 497 | stub_connection_if_needed(connection, 'mysql') do 498 | mysql_worker = BulkInsert::Worker.new( 499 | connection, 500 | Testing.table_name, 501 | 'id', 502 | %w(greeting age happy created_at updated_at color), 503 | 500, # batch size 504 | false, # ignore 505 | true # update_duplicates 506 | ) 507 | mysql_worker.add ["Yo", 15, false, nil, nil] 508 | 509 | assert_statement_adapter mysql_worker, 'BulkInsert::StatementAdapters::MySQLAdapter' 510 | assert_equal mysql_worker.compose_insert_query, "INSERT INTO `testings` (`greeting`,`age`,`happy`,`created_at`,`updated_at`,`color`) VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)" 511 | end 512 | end 513 | 514 | def assert_statement_adapter(worker, adapter_name) 515 | assert_equal worker.instance_variable_get(:@statement_adapter).class.to_s, adapter_name 516 | end 517 | end 518 | -------------------------------------------------------------------------------- /test/bulk_insert_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class BulkInsertTest < ActiveSupport::TestCase 4 | test "bulk_insert without block should return worker" do 5 | result = Testing.bulk_insert 6 | assert_kind_of BulkInsert::Worker, result 7 | end 8 | 9 | test "bulk_insert with block should yield worker" do 10 | result = nil 11 | Testing.bulk_insert { |worker| result = worker } 12 | assert_kind_of BulkInsert::Worker, result 13 | end 14 | 15 | test "bulk_insert with block should save automatically" do 16 | assert_difference "Testing.count", 1 do 17 | Testing.bulk_insert do |worker| 18 | worker.add greeting: "Hello" 19 | end 20 | end 21 | end 22 | 23 | test "worker should not have any result sets without option for returning primary keys" do 24 | worker = Testing.bulk_insert 25 | worker.add greeting: "hello" 26 | worker.save! 27 | assert_empty worker.result_sets 28 | end 29 | 30 | test "with option to return primary keys, worker should have result sets" do 31 | worker = Testing.bulk_insert(return_primary_keys: true) 32 | worker.add greeting: "yo" 33 | 34 | # return_primary_keys is not supported for mysql and rails < 5 35 | # this test ensures that the case is covered in the CI and handled as expected 36 | if ActiveRecord::VERSION::STRING < "5.0.0" && worker.adapter_name =~ /^mysql/i 37 | error = assert_raise(ArgumentError) { worker.save! } 38 | assert_equal error.message, "BulkInsert does not support @return_primary_keys for mysql and rails < 5" 39 | else 40 | worker.save! 41 | assert_equal 1, worker.result_sets.count 42 | end 43 | end 44 | 45 | test "bulk_insert with array should save the array immediately" do 46 | assert_difference "Testing.count", 2 do 47 | Testing.bulk_insert values: [ 48 | [ "Hello", 15, true ], 49 | { greeting: "Hey", age: 20, happy: false } 50 | ] 51 | end 52 | end 53 | 54 | test "default_bulk_columns should return all columns without id" do 55 | default_columns = %w(greeting age happy created_at updated_at color) 56 | 57 | assert_equal Testing.default_bulk_columns, default_columns 58 | end 59 | 60 | end 61 | -------------------------------------------------------------------------------- /test/connection_mocks.rb: -------------------------------------------------------------------------------- 1 | module ConnectionMocks 2 | DOUBLE_QUOTE_PROC = Proc.new do |value, *_column| 3 | return value unless value.is_a? String 4 | "\"#{value}\"" 5 | end 6 | 7 | BACKTICK_QUOTE_PROC = Proc.new do |value, *_column| 8 | return value unless value.is_a? String 9 | '`' + value + '`' 10 | end 11 | 12 | BOOLEAN_VALUE_QUOTE_PROC = Proc.new do |value, *_column| 13 | case value 14 | when String 15 | "'" + value + "'" 16 | when TrueClass 17 | 'TRUE' 18 | when FalseClass 19 | 'FALSE' 20 | when NilClass 21 | 'NULL' 22 | else 23 | value 24 | end 25 | end 26 | 27 | LITERAL_BOOLEAN_VALUE_QUOTE_PROC = Proc.new do |value, *_column| 28 | case value 29 | when String 30 | "'" + value + "'" 31 | when TrueClass 32 | "'t'" 33 | when FalseClass 34 | "'f'" 35 | when NilClass 36 | 'NULL' 37 | else 38 | value 39 | end 40 | end 41 | 42 | DEFAULT_VALUE_QUOTE_PROC = Proc.new do |value, *_column| 43 | case value 44 | when String 45 | "'" + value + "'" 46 | when TrueClass 47 | 1 48 | when FalseClass 49 | 0 50 | when NilClass 51 | 'NULL' 52 | else 53 | value 54 | end 55 | end 56 | 57 | ColumnMock = Struct.new(:name, :default) 58 | COLUMNS_MOCK_PROC = Proc.new do |*_table_name| 59 | %w(id greeting age happy created_at updated_at color).zip( 60 | [nil, nil, nil, nil, nil, nil, "chartreuse"] 61 | ).map do |column_name, default| 62 | ColumnMock.new(column_name, default) 63 | end 64 | end 65 | 66 | MockTypeSerialize = Struct.new(:column) do 67 | def serialize(value); value; end 68 | end 69 | CAST_COLUMN_MOCK_PROC = Proc.new do |column| 70 | MockTypeSerialize.new(column) 71 | end 72 | 73 | def stub_connection_if_needed(connection, adapter_name) 74 | raise "You need to provide a block" unless block_given? 75 | if connection.adapter_name == adapter_name 76 | yield 77 | else 78 | common_mocks(connection, adapter_name) do 79 | case adapter_name 80 | when /^mysql/i 81 | mock_mysql_connection(connection, adapter_name) do 82 | yield 83 | end 84 | when /\APost(?:greSQL|GIS)/i 85 | mock_postgresql_connection(connection, adapter_name) do 86 | yield 87 | end 88 | else 89 | connection.stub :quote_table_name, DOUBLE_QUOTE_PROC do 90 | connection.stub :quote_column_name, DOUBLE_QUOTE_PROC do 91 | connection.stub :quote, DEFAULT_VALUE_QUOTE_PROC do 92 | yield 93 | end 94 | end 95 | end 96 | end 97 | end 98 | end 99 | end 100 | 101 | def common_mocks(connection, adapter_name) 102 | connection.stub :adapter_name, adapter_name do 103 | connection.stub :columns, COLUMNS_MOCK_PROC do 104 | if ActiveRecord::VERSION::STRING >= "5.0.0" 105 | connection.stub :lookup_cast_type_from_column, CAST_COLUMN_MOCK_PROC do 106 | yield 107 | end 108 | else 109 | yield 110 | end 111 | end 112 | end 113 | end 114 | 115 | def mock_mysql_connection(connection, adapter_name) 116 | connection.stub :quote_table_name, BACKTICK_QUOTE_PROC do 117 | connection.stub :quote_column_name, BACKTICK_QUOTE_PROC do 118 | connection.stub :quote, BOOLEAN_VALUE_QUOTE_PROC do 119 | yield 120 | end 121 | end 122 | end 123 | end 124 | 125 | def mock_postgresql_connection(connection, adapter_name) 126 | connection.stub :quote_table_name, DOUBLE_QUOTE_PROC do 127 | connection.stub :quote_column_name, DOUBLE_QUOTE_PROC do 128 | if ActiveRecord::VERSION::STRING >= "5.0.0" 129 | connection.stub :quote, BOOLEAN_VALUE_QUOTE_PROC do 130 | yield 131 | end 132 | else 133 | connection.stub :quote, LITERAL_BOOLEAN_VALUE_QUOTE_PROC do 134 | yield 135 | end 136 | end 137 | end 138 | end 139 | end 140 | end 141 | -------------------------------------------------------------------------------- /test/dummy/README.rdoc: -------------------------------------------------------------------------------- 1 | == README 2 | 3 | This README would normally document whatever steps are necessary to get the 4 | application up and running. 5 | 6 | Things you may want to cover: 7 | 8 | * Ruby version 9 | 10 | * System dependencies 11 | 12 | * Configuration 13 | 14 | * Database creation 15 | 16 | * Database initialization 17 | 18 | * How to run the test suite 19 | 20 | * Services (job queues, cache servers, search engines, etc.) 21 | 22 | * Deployment instructions 23 | 24 | * ... 25 | 26 | 27 | Please feel free to use a different markup language if you do not plan to run 28 | rake doc:app. 29 | -------------------------------------------------------------------------------- /test/dummy/Rakefile: -------------------------------------------------------------------------------- 1 | # Add your own tasks in files placed in lib/tasks ending in .rake, 2 | # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. 3 | 4 | require File.expand_path('../config/application', __FILE__) 5 | 6 | Rails.application.load_tasks 7 | -------------------------------------------------------------------------------- /test/dummy/app/assets/config/manifest.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamis/bulk_insert/ab5db0873098701904ac2fcdcab86e417d342895/test/dummy/app/assets/config/manifest.js -------------------------------------------------------------------------------- /test/dummy/app/assets/images/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamis/bulk_insert/ab5db0873098701904ac2fcdcab86e417d342895/test/dummy/app/assets/images/.keep -------------------------------------------------------------------------------- /test/dummy/app/assets/javascripts/application.js: -------------------------------------------------------------------------------- 1 | // This is a manifest file that'll be compiled into application.js, which will include all the files 2 | // listed below. 3 | // 4 | // Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts, 5 | // or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path. 6 | // 7 | // It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the 8 | // compiled file. 9 | // 10 | // Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details 11 | // about supported directives. 12 | // 13 | //= require_tree . 14 | -------------------------------------------------------------------------------- /test/dummy/app/assets/stylesheets/application.css: -------------------------------------------------------------------------------- 1 | /* 2 | * This is a manifest file that'll be compiled into application.css, which will include all the files 3 | * listed below. 4 | * 5 | * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets, 6 | * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path. 7 | * 8 | * You're free to add application-wide styles to this file and they'll appear at the bottom of the 9 | * compiled file so the styles you add here take precedence over styles defined in any styles 10 | * defined in the other CSS/SCSS files in this directory. It is generally better to create a new 11 | * file per style scope. 12 | * 13 | *= require_tree . 14 | *= require_self 15 | */ 16 | -------------------------------------------------------------------------------- /test/dummy/app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | class ApplicationController < ActionController::Base 2 | # Prevent CSRF attacks by raising an exception. 3 | # For APIs, you may want to use :null_session instead. 4 | protect_from_forgery with: :exception 5 | end 6 | -------------------------------------------------------------------------------- /test/dummy/app/controllers/concerns/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamis/bulk_insert/ab5db0873098701904ac2fcdcab86e417d342895/test/dummy/app/controllers/concerns/.keep -------------------------------------------------------------------------------- /test/dummy/app/helpers/application_helper.rb: -------------------------------------------------------------------------------- 1 | module ApplicationHelper 2 | end 3 | -------------------------------------------------------------------------------- /test/dummy/app/mailers/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamis/bulk_insert/ab5db0873098701904ac2fcdcab86e417d342895/test/dummy/app/mailers/.keep -------------------------------------------------------------------------------- /test/dummy/app/models/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamis/bulk_insert/ab5db0873098701904ac2fcdcab86e417d342895/test/dummy/app/models/.keep -------------------------------------------------------------------------------- /test/dummy/app/models/concerns/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamis/bulk_insert/ab5db0873098701904ac2fcdcab86e417d342895/test/dummy/app/models/concerns/.keep -------------------------------------------------------------------------------- /test/dummy/app/models/testing.rb: -------------------------------------------------------------------------------- 1 | class Testing < ActiveRecord::Base 2 | end 3 | -------------------------------------------------------------------------------- /test/dummy/app/views/layouts/application.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Dummy 5 | <%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track' => true %> 6 | <%= javascript_include_tag 'application', 'data-turbolinks-track' => true %> 7 | <%= csrf_meta_tags %> 8 | 9 | 10 | 11 | <%= yield %> 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /test/dummy/bin/bundle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 3 | load Gem.bin_path('bundler', 'bundle') 4 | -------------------------------------------------------------------------------- /test/dummy/bin/rails: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_PATH = File.expand_path('../../config/application', __FILE__) 3 | require_relative '../config/boot' 4 | require 'rails/commands' 5 | -------------------------------------------------------------------------------- /test/dummy/bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require_relative '../config/boot' 3 | require 'rake' 4 | Rake.application.run 5 | -------------------------------------------------------------------------------- /test/dummy/bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'pathname' 3 | 4 | # path to your application root. 5 | APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) 6 | 7 | Dir.chdir APP_ROOT do 8 | # This script is a starting point to setup your application. 9 | # Add necessary setup steps to this file: 10 | 11 | puts "== Installing dependencies ==" 12 | system "gem install bundler --conservative" 13 | system "bundle check || bundle install" 14 | 15 | # puts "\n== Copying sample files ==" 16 | # unless File.exist?("config/database.yml") 17 | # system "cp config/database.yml.sample config/database.yml" 18 | # end 19 | 20 | puts "\n== Preparing database ==" 21 | system "bin/rake db:setup" 22 | 23 | puts "\n== Removing old logs and tempfiles ==" 24 | system "rm -f log/*" 25 | system "rm -rf tmp/cache" 26 | 27 | puts "\n== Restarting application server ==" 28 | system "touch tmp/restart.txt" 29 | end 30 | -------------------------------------------------------------------------------- /test/dummy/config.ru: -------------------------------------------------------------------------------- 1 | # This file is used by Rack-based servers to start the application. 2 | 3 | require ::File.expand_path('../config/environment', __FILE__) 4 | run Rails.application 5 | -------------------------------------------------------------------------------- /test/dummy/config/application.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../boot', __FILE__) 2 | 3 | require 'rails/all' 4 | 5 | Bundler.require(*Rails.groups) 6 | require "bulk_insert" 7 | 8 | module Dummy 9 | class Application < Rails::Application 10 | # Settings in config/environments/* take precedence over those specified here. 11 | # Application configuration should go into files in config/initializers 12 | # -- all .rb files in that directory are automatically loaded. 13 | 14 | # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone. 15 | # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC. 16 | # config.time_zone = 'Central Time (US & Canada)' 17 | 18 | # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded. 19 | # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] 20 | # config.i18n.default_locale = :de 21 | RAILS_VERSION = Gem.loaded_specs['rails'].version 22 | 23 | # Patch MySQL to execute tests 24 | # Mysql2::Error: All parts of a PRIMARY KEY must be NOT NULL; if you need NULL in a key, use UNIQUE instead 25 | if RAILS_VERSION < Gem::Version.new('4.0.0') 26 | # https://stackoverflow.com/a/40758542/5687152 27 | require 'active_record/connection_adapters/mysql2_adapter' 28 | 29 | class ActiveRecord::ConnectionAdapters::Mysql2Adapter 30 | NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY" 31 | end 32 | end 33 | 34 | # Patch SQLite to support multiple rails versions with the same app 35 | # Tests are written assuming booleans as integers 36 | # https://github.com/rails/rails/commit/a18cf23a9cbcbeed61e8049442640c7153e0a8fb 37 | if RAILS_VERSION < Gem::Version.new('5.2.0') 38 | # https://github.com/rails/rails/commit/52e050ed00b023968fecda82f19a858876a7c435 39 | require 'active_record/connection_adapters/sqlite3_adapter' 40 | ActiveRecord::ConnectionAdapters::SQLite3Adapter.class_eval do 41 | class_attribute :represent_boolean_as_integer, default: false 42 | # end 43 | # ActiveRecord::ConnectionAdapters::SQLite3::Quoting.module_eval do 44 | def quoted_true 45 | ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? "1".freeze : "'t'".freeze 46 | end 47 | 48 | def unquoted_true 49 | ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? 1 : "t".freeze 50 | end 51 | 52 | def quoted_false 53 | ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? "0".freeze : "'f'".freeze 54 | end 55 | 56 | def unquoted_false 57 | ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? 0 : "f".freeze 58 | end 59 | end 60 | 61 | ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer = true 62 | # https://github.com/rails/rails/commit/f59b08119bc0c01a00561d38279b124abc82561b 63 | elsif RAILS_VERSION < Gem::Version.new('6.1.0') 64 | config.active_record.sqlite3.represent_boolean_as_integer = true 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /test/dummy/config/boot.rb: -------------------------------------------------------------------------------- 1 | # Set up gems listed in the Gemfile. 2 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__) 3 | 4 | require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE']) 5 | $LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__) 6 | -------------------------------------------------------------------------------- /test/dummy/config/database.yml: -------------------------------------------------------------------------------- 1 | sqlite: &sqlite 2 | adapter: sqlite3 3 | database: db/test.sqlite3 4 | pool: 5 5 | timeout: 5000 6 | 7 | mysql: &mysql 8 | adapter: mysql2 9 | database: bulk_insert_test 10 | encoding: utf8 11 | pool: 5 12 | timeout: 5000 13 | username: travis 14 | 15 | postgresql: &postgresql 16 | adapter: postgresql 17 | database: bulk_insert_test 18 | username: postgres 19 | 20 | test: 21 | <<: *<%= ENV['DB_ADAPTER'] || 'sqlite' %> 22 | -------------------------------------------------------------------------------- /test/dummy/config/environment.rb: -------------------------------------------------------------------------------- 1 | # Load the Rails application. 2 | require File.expand_path('../application', __FILE__) 3 | 4 | # Initialize the Rails application. 5 | Rails.application.initialize! 6 | -------------------------------------------------------------------------------- /test/dummy/config/environments/development.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # In the development environment your application's code is reloaded on 5 | # every request. This slows down response time but is perfect for development 6 | # since you don't have to restart the web server when you make code changes. 7 | config.cache_classes = false 8 | 9 | # Do not eager load code on boot. 10 | config.eager_load = false 11 | 12 | # Show full error reports and disable caching. 13 | config.consider_all_requests_local = true 14 | config.action_controller.perform_caching = false 15 | 16 | # Don't care if the mailer can't send. 17 | config.action_mailer.raise_delivery_errors = false 18 | 19 | # Print deprecation notices to the Rails logger. 20 | config.active_support.deprecation = :log 21 | 22 | # Raise an error on page load if there are pending migrations. 23 | config.active_record.migration_error = :page_load 24 | 25 | # Debug mode disables concatenation and preprocessing of assets. 26 | # This option may cause significant delays in view rendering with a large 27 | # number of complex assets. 28 | config.assets.debug = true 29 | 30 | # Asset digests allow you to set far-future HTTP expiration dates on all assets, 31 | # yet still be able to expire them through the digest params. 32 | config.assets.digest = true 33 | 34 | # Adds additional error checking when serving assets at runtime. 35 | # Checks for improperly declared sprockets dependencies. 36 | # Raises helpful error messages. 37 | config.assets.raise_runtime_errors = true 38 | 39 | # Raises error for missing translations 40 | # config.action_view.raise_on_missing_translations = true 41 | end 42 | -------------------------------------------------------------------------------- /test/dummy/config/environments/production.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # Code is not reloaded between requests. 5 | config.cache_classes = true 6 | 7 | # Eager load code on boot. This eager loads most of Rails and 8 | # your application in memory, allowing both threaded web servers 9 | # and those relying on copy on write to perform better. 10 | # Rake tasks automatically ignore this option for performance. 11 | config.eager_load = true 12 | 13 | # Full error reports are disabled and caching is turned on. 14 | config.consider_all_requests_local = false 15 | config.action_controller.perform_caching = true 16 | 17 | # Enable Rack::Cache to put a simple HTTP cache in front of your application 18 | # Add `rack-cache` to your Gemfile before enabling this. 19 | # For large-scale production use, consider using a caching reverse proxy like 20 | # NGINX, varnish or squid. 21 | # config.action_dispatch.rack_cache = true 22 | 23 | # Disable serving static files from the `/public` folder by default since 24 | # Apache or NGINX already handles this. 25 | config.serve_static_files = ENV['RAILS_SERVE_STATIC_FILES'].present? 26 | 27 | # Compress JavaScripts and CSS. 28 | config.assets.js_compressor = :uglifier 29 | # config.assets.css_compressor = :sass 30 | 31 | # Do not fallback to assets pipeline if a precompiled asset is missed. 32 | config.assets.compile = false 33 | 34 | # Asset digests allow you to set far-future HTTP expiration dates on all assets, 35 | # yet still be able to expire them through the digest params. 36 | config.assets.digest = true 37 | 38 | # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb 39 | 40 | # Specifies the header that your server uses for sending files. 41 | # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache 42 | # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX 43 | 44 | # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. 45 | # config.force_ssl = true 46 | 47 | # Use the lowest log level to ensure availability of diagnostic information 48 | # when problems arise. 49 | config.log_level = :debug 50 | 51 | # Prepend all log lines with the following tags. 52 | # config.log_tags = [ :subdomain, :uuid ] 53 | 54 | # Use a different logger for distributed setups. 55 | # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) 56 | 57 | # Use a different cache store in production. 58 | # config.cache_store = :mem_cache_store 59 | 60 | # Enable serving of images, stylesheets, and JavaScripts from an asset server. 61 | # config.action_controller.asset_host = 'http://assets.example.com' 62 | 63 | # Ignore bad email addresses and do not raise email delivery errors. 64 | # Set this to true and configure the email server for immediate delivery to raise delivery errors. 65 | # config.action_mailer.raise_delivery_errors = false 66 | 67 | # Enable locale fallbacks for I18n (makes lookups for any locale fall back to 68 | # the I18n.default_locale when a translation cannot be found). 69 | config.i18n.fallbacks = true 70 | 71 | # Send deprecation notices to registered listeners. 72 | config.active_support.deprecation = :notify 73 | 74 | # Use default logging formatter so that PID and timestamp are not suppressed. 75 | config.log_formatter = ::Logger::Formatter.new 76 | 77 | # Do not dump schema after migrations. 78 | config.active_record.dump_schema_after_migration = false 79 | end 80 | -------------------------------------------------------------------------------- /test/dummy/config/environments/test.rb: -------------------------------------------------------------------------------- 1 | Dummy::Application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # The test environment is used exclusively to run your application's 5 | # test suite. You never need to work with it otherwise. Remember that 6 | # your test database is "scratch space" for the test suite and is wiped 7 | # and recreated between test runs. Don't rely on the data there! 8 | config.cache_classes = true 9 | 10 | # Do not eager load code on boot. This avoids loading your whole application 11 | # just for the purpose of running a single test. If you are using a tool that 12 | # preloads Rails for running tests, you may have to set it to true. 13 | config.eager_load = false 14 | 15 | # Configure static file server for tests with Cache-Control for performance. 16 | config.serve_static_files = true 17 | config.static_cache_control = 'public, max-age=3600' 18 | 19 | # Show full error reports and disable caching. 20 | config.consider_all_requests_local = true 21 | config.action_controller.perform_caching = false 22 | 23 | # Raise exceptions instead of rendering exception templates. 24 | config.action_dispatch.show_exceptions = false 25 | 26 | # Disable request forgery protection in test environment. 27 | config.action_controller.allow_forgery_protection = false 28 | 29 | # Tell Action Mailer not to deliver emails to the real world. 30 | # The :test delivery method accumulates sent emails in the 31 | # ActionMailer::Base.deliveries array. 32 | config.action_mailer.delivery_method = :test 33 | 34 | # Randomize the order test cases are executed. 35 | config.active_support.test_order = :random 36 | 37 | # Print deprecation notices to the stderr. 38 | config.active_support.deprecation = :stderr 39 | 40 | # Raises error for missing translations 41 | # config.action_view.raise_on_missing_translations = true 42 | end 43 | -------------------------------------------------------------------------------- /test/dummy/config/initializers/assets.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Version of your assets, change this if you want to expire all your assets. 4 | Rails.application.config.assets.version = '1.0' 5 | 6 | # Add additional assets to the asset load path 7 | # Rails.application.config.assets.paths << Emoji.images_path 8 | 9 | # Precompile additional assets. 10 | # application.js, application.css, and all non-JS/CSS in app/assets folder are already added. 11 | # Rails.application.config.assets.precompile += %w( search.js ) 12 | -------------------------------------------------------------------------------- /test/dummy/config/initializers/backtrace_silencers.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. 4 | # Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } 5 | 6 | # You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. 7 | # Rails.backtrace_cleaner.remove_silencers! 8 | -------------------------------------------------------------------------------- /test/dummy/config/initializers/cookies_serializer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | Rails.application.config.action_dispatch.cookies_serializer = :json 4 | -------------------------------------------------------------------------------- /test/dummy/config/initializers/filter_parameter_logging.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Configure sensitive parameters which will be filtered from the log file. 4 | Rails.application.config.filter_parameters += [:password] 5 | -------------------------------------------------------------------------------- /test/dummy/config/initializers/inflections.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new inflection rules using the following format. Inflections 4 | # are locale specific, and you may define rules for as many different 5 | # locales as you wish. All of these examples are active by default: 6 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 7 | # inflect.plural /^(ox)$/i, '\1en' 8 | # inflect.singular /^(ox)en/i, '\1' 9 | # inflect.irregular 'person', 'people' 10 | # inflect.uncountable %w( fish sheep ) 11 | # end 12 | 13 | # These inflection rules are supported but not enabled by default: 14 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 15 | # inflect.acronym 'RESTful' 16 | # end 17 | -------------------------------------------------------------------------------- /test/dummy/config/initializers/mime_types.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new mime types for use in respond_to blocks: 4 | # Mime::Type.register "text/richtext", :rtf 5 | -------------------------------------------------------------------------------- /test/dummy/config/initializers/session_store.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | Rails.application.config.session_store :cookie_store, key: '_dummy_session' 4 | -------------------------------------------------------------------------------- /test/dummy/config/initializers/wrap_parameters.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # This file contains settings for ActionController::ParamsWrapper which 4 | # is enabled by default. 5 | 6 | # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array. 7 | ActiveSupport.on_load(:action_controller) do 8 | wrap_parameters format: [:json] if respond_to?(:wrap_parameters) 9 | end 10 | 11 | # To enable root element in JSON for ActiveRecord objects. 12 | # ActiveSupport.on_load(:active_record) do 13 | # self.include_root_in_json = true 14 | # end 15 | -------------------------------------------------------------------------------- /test/dummy/config/locales/en.yml: -------------------------------------------------------------------------------- 1 | # Files in the config/locales directory are used for internationalization 2 | # and are automatically loaded by Rails. If you want to use locales other 3 | # than English, add the necessary files in this directory. 4 | # 5 | # To use the locales, use `I18n.t`: 6 | # 7 | # I18n.t 'hello' 8 | # 9 | # In views, this is aliased to just `t`: 10 | # 11 | # <%= t('hello') %> 12 | # 13 | # To use a different locale, set it with `I18n.locale`: 14 | # 15 | # I18n.locale = :es 16 | # 17 | # This would use the information in config/locales/es.yml. 18 | # 19 | # To learn more, please read the Rails Internationalization guide 20 | # available at http://guides.rubyonrails.org/i18n.html. 21 | 22 | en: 23 | hello: "Hello world" 24 | -------------------------------------------------------------------------------- /test/dummy/config/routes.rb: -------------------------------------------------------------------------------- 1 | Rails.application.routes.draw do 2 | # The priority is based upon order of creation: first created -> highest priority. 3 | # See how all your routes lay out with "rake routes". 4 | 5 | # You can have the root of your site routed with "root" 6 | # root 'welcome#index' 7 | 8 | # Example of regular route: 9 | # get 'products/:id' => 'catalog#view' 10 | 11 | # Example of named route that can be invoked with purchase_url(id: product.id) 12 | # get 'products/:id/purchase' => 'catalog#purchase', as: :purchase 13 | 14 | # Example resource route (maps HTTP verbs to controller actions automatically): 15 | # resources :products 16 | 17 | # Example resource route with options: 18 | # resources :products do 19 | # member do 20 | # get 'short' 21 | # post 'toggle' 22 | # end 23 | # 24 | # collection do 25 | # get 'sold' 26 | # end 27 | # end 28 | 29 | # Example resource route with sub-resources: 30 | # resources :products do 31 | # resources :comments, :sales 32 | # resource :seller 33 | # end 34 | 35 | # Example resource route with more complex sub-resources: 36 | # resources :products do 37 | # resources :comments 38 | # resources :sales do 39 | # get 'recent', on: :collection 40 | # end 41 | # end 42 | 43 | # Example resource route with concerns: 44 | # concern :toggleable do 45 | # post 'toggle' 46 | # end 47 | # resources :posts, concerns: :toggleable 48 | # resources :photos, concerns: :toggleable 49 | 50 | # Example resource route within a namespace: 51 | # namespace :admin do 52 | # # Directs /admin/products/* to Admin::ProductsController 53 | # # (app/controllers/admin/products_controller.rb) 54 | # resources :products 55 | # end 56 | end 57 | -------------------------------------------------------------------------------- /test/dummy/config/secrets.yml: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Your secret key is used for verifying the integrity of signed cookies. 4 | # If you change this key, all old signed cookies will become invalid! 5 | 6 | # Make sure the secret is at least 30 characters and all random, 7 | # no regular words or you'll be exposed to dictionary attacks. 8 | # You can use `rake secret` to generate a secure secret key. 9 | 10 | # Make sure the secrets in this file are kept private 11 | # if you're sharing your code publicly. 12 | 13 | development: 14 | secret_key_base: f6486be44b975842a9304a32397ca8d99b1d3e9fecbede92ffb16c60091d3d16a23092e3a53ae3561c1c0f7302a3549e48536f27e240d4b948d125a846171f2a 15 | 16 | test: 17 | secret_key_base: 344e414a6671f7f8b5a409072134ace55a41f0f59e9f044119419ddcac2d537405a6558173af09ec14e0c11e77cdd0a85e33c3d28fbc022424b3e86bd4ff35db 18 | 19 | # Do not keep production secrets in the repository, 20 | # instead read values from the environment. 21 | production: 22 | secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> 23 | -------------------------------------------------------------------------------- /test/dummy/db/migrate/20151008181535_create_testings.rb: -------------------------------------------------------------------------------- 1 | class CreateTestings < ActiveRecord::Migration 2 | def change 3 | create_table :testings do |t| 4 | t.string :greeting 5 | t.integer :age 6 | t.boolean :happy 7 | 8 | t.timestamps null: false 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /test/dummy/db/migrate/20151028194232_add_default_value.rb: -------------------------------------------------------------------------------- 1 | class AddDefaultValue < ActiveRecord::Migration 2 | def change 3 | add_column :testings, :color, :string, default: "chartreuse" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /test/dummy/db/schema.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | # This file is auto-generated from the current state of the database. Instead 3 | # of editing this file, please use the migrations feature of Active Record to 4 | # incrementally modify your database, and then regenerate this schema definition. 5 | # 6 | # Note that this schema.rb definition is the authoritative source for your 7 | # database schema. If you need to create the application database on another 8 | # system, you should be using db:schema:load, not running all the migrations 9 | # from scratch. The latter is a flawed and unsustainable approach (the more migrations 10 | # you'll amass, the slower it'll run and the greater likelihood for issues). 11 | # 12 | # It's strongly recommended that you check this file into your version control system. 13 | 14 | ActiveRecord::Schema.define(version: 20151028194232) do 15 | 16 | create_table "testings", force: :cascade do |t| 17 | t.string "greeting" 18 | t.integer "age" 19 | t.boolean "happy" 20 | t.datetime "created_at", null: false 21 | t.datetime "updated_at", null: false 22 | t.string "color", default: "chartreuse" 23 | end 24 | 25 | end 26 | -------------------------------------------------------------------------------- /test/dummy/lib/assets/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamis/bulk_insert/ab5db0873098701904ac2fcdcab86e417d342895/test/dummy/lib/assets/.keep -------------------------------------------------------------------------------- /test/dummy/log/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamis/bulk_insert/ab5db0873098701904ac2fcdcab86e417d342895/test/dummy/log/.keep -------------------------------------------------------------------------------- /test/dummy/public/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The page you were looking for doesn't exist (404) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The page you were looking for doesn't exist.

62 |

You may have mistyped the address or the page may have moved.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /test/dummy/public/422.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The change you wanted was rejected (422) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The change you wanted was rejected.

62 |

Maybe you tried to change something you didn't have access to.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /test/dummy/public/500.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | We're sorry, but something went wrong (500) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

We're sorry, but something went wrong.

62 |
63 |

If you are the application owner check the logs for more information.

64 |
65 | 66 | 67 | -------------------------------------------------------------------------------- /test/dummy/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamis/bulk_insert/ab5db0873098701904ac2fcdcab86e417d342895/test/dummy/public/favicon.ico -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | # Configure Rails Environment 2 | ENV["RAILS_ENV"] = "test" 3 | 4 | require File.expand_path("../../test/dummy/config/environment.rb", __FILE__) 5 | ActiveRecord::Migrator.migrations_paths = [File.expand_path("../../test/dummy/db/migrate", __FILE__)] 6 | require "rails/test_help" 7 | 8 | # Filter out Minitest backtrace while allowing backtrace from other libraries 9 | # to be shown. 10 | Minitest.backtrace_filter = Minitest::BacktraceFilter.new 11 | 12 | # Load support files 13 | Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f } 14 | 15 | # Load fixtures from the engine 16 | if ActiveSupport::TestCase.respond_to?(:fixture_path=) 17 | ActiveSupport::TestCase.fixture_path = File.expand_path("../fixtures", __FILE__) 18 | ActiveSupport::TestCase.fixtures :all 19 | end 20 | --------------------------------------------------------------------------------