├── .rspec ├── Rakefile ├── lib ├── active_importer │ ├── version.rb │ └── base.rb └── active_importer.rb ├── Gemfile ├── spec ├── support │ ├── active_record │ │ ├── models.rb │ │ └── schema.rb │ ├── spreadsheet.rb │ └── employee_importer.rb ├── spec_helper.rb └── active_importer │ └── base_spec.rb ├── .gitignore ├── LICENSE.txt ├── active_importer.gemspec └── README.md /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format progress 3 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | -------------------------------------------------------------------------------- /lib/active_importer/version.rb: -------------------------------------------------------------------------------- 1 | module ActiveImporter 2 | VERSION = "0.2.6" 3 | end 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in active_importer.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /lib/active_importer.rb: -------------------------------------------------------------------------------- 1 | require "active_importer/version" 2 | 3 | module ActiveImporter 4 | autoload :Base, 'active_importer/base' 5 | end 6 | -------------------------------------------------------------------------------- /spec/support/active_record/models.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../schema', __FILE__) 2 | 3 | class Employee < ::ActiveRecord::Base 4 | validates_exclusion_of :name, in: ['Invalid'] 5 | end 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | -------------------------------------------------------------------------------- /spec/support/spreadsheet.rb: -------------------------------------------------------------------------------- 1 | class Spreadsheet 2 | def initialize(data) 3 | data = { "Default" => data } unless data.is_a?(Hash) 4 | @data = data 5 | end 6 | 7 | def sheets 8 | @sheets ||= @data.keys 9 | end 10 | 11 | def default_sheet 12 | @default_sheet ||= sheets.first 13 | end 14 | 15 | def default_sheet=(value) 16 | raise "Invalid sheet '#{value}'" unless sheets.include?(value) 17 | @default_sheet = value 18 | end 19 | 20 | def last_row 21 | @data[default_sheet].count 22 | end 23 | 24 | def row(index) 25 | @data[default_sheet][index-1] 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /spec/support/active_record/schema.rb: -------------------------------------------------------------------------------- 1 | require 'active_record' 2 | require 'logger' 3 | 4 | ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:') 5 | ActiveRecord::Base.logger = Logger.new('/dev/null') 6 | ActiveRecord::Migration.verbose = false 7 | 8 | ActiveRecord::Schema.define do 9 | create_table :employees, :force => true do |t| 10 | t.column :name, :string 11 | t.column :birth_date, :string 12 | t.column :department_id, :integer 13 | t.column :unused_field, :string 14 | t.column :created_at, :datetime 15 | t.column :updated_at, :datetime 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'active_importer' 2 | 3 | # Require files in spec/support 4 | Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f } 5 | 6 | # This file was generated by the `rspec --init` command. Conventionally, all 7 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. 8 | # Require this file using `require "spec_helper"` to ensure that it is only 9 | # loaded once. 10 | # 11 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 12 | RSpec.configure do |config| 13 | config.run_all_when_everything_filtered = true 14 | 15 | config.expect_with :rspec do |c| 16 | c.syntax = :expect 17 | end 18 | 19 | # Run specs in random order to surface order dependencies. If you find an 20 | # order dependency and want to debug it, you can fix the order by providing 21 | # the seed, which is printed after each run. 22 | # --seed 1234 23 | config.order = 'random' 24 | end 25 | 26 | I18n.enforce_available_locales = false 27 | -------------------------------------------------------------------------------- /spec/support/employee_importer.rb: -------------------------------------------------------------------------------- 1 | class EmployeeBaseImporter < ActiveImporter::Base 2 | on(:import_finished) { base_import_finished } 3 | 4 | skip_rows_if do 5 | row['Name'] == 'BaseSkip' 6 | end 7 | 8 | private 9 | 10 | def base_import_finished 11 | end 12 | end 13 | 14 | class EmployeeImporter < EmployeeBaseImporter 15 | imports Employee 16 | 17 | column 'Name', :name 18 | column 'Birth Date', :birth_date 19 | column 'Manager' 20 | column 'Unused', :unused_field, optional: true 21 | column 'Extra', optional: true 22 | column ' Department ', :department_id do |value| 23 | find_department(value) 24 | end 25 | 26 | on :row_processing do 27 | abort!('Row cannot be processed') if row['Name'] == 'Abort' 28 | end 29 | 30 | skip_rows_if do 31 | row['Name'] == 'Skip' 32 | end 33 | 34 | def find_department(name) 35 | name.length # Quick dummy way to get an integer out of a string 36 | end 37 | 38 | ActiveImporter::Base::EVENTS.each do |event_name| 39 | define_method(event_name) {} 40 | on(event_name) { send event_name } 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Ernesto Garcia 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /active_importer.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'active_importer/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "active_importer" 8 | spec.version = ActiveImporter::VERSION 9 | spec.authors = ["Ernesto Garcia"] 10 | spec.email = ["gnapse@gmail.com"] 11 | spec.description = %q{Import tabular data from spreadsheets or similar sources into data models} 12 | spec.summary = %q{Import tabular data into data models} 13 | spec.homepage = "http://continuum.github.io/active_importer/" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_dependency "roo" 22 | 23 | spec.add_development_dependency "bundler", "~> 1.3" 24 | spec.add_development_dependency "rake" 25 | spec.add_development_dependency "rspec", ">= 2.2.0" 26 | 27 | spec.add_development_dependency "sqlite3" 28 | spec.add_development_dependency "activerecord" 29 | end 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ActiveImporter 2 | [![Gitter](https://badges.gitter.im/Join Chat.svg)](https://gitter.im/continuum/active_importer?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 3 | 4 | ## ARCHIVED 5 | 6 | Define importers that load tabular data from spreadsheets or CSV files into any ActiveRecord-like ORM. 7 | 8 | ## Installation 9 | 10 | Add this line to your application's Gemfile: 11 | 12 | gem 'active_importer' 13 | 14 | And then execute: 15 | 16 | $ bundle 17 | 18 | Or install it yourself as: 19 | 20 | $ gem install active_importer 21 | 22 | ## Usage 23 | 24 | Define classes that you instruct on how to import data into data models. 25 | 26 | ```ruby 27 | class EmployeeImporter < ActiveImporter::Base 28 | imports Employee 29 | 30 | column 'First name', :first_name 31 | column 'Last name', :last_name 32 | column 'Department', :department do |department_name| 33 | Department.find_by(name: department_name) 34 | end 35 | end 36 | ``` 37 | 38 | The importer defines what data model it imports data into, and how columns in 39 | the data source map to fields in the model. Also, by providing a block, the 40 | source value can be processed before being stored, as shown with the 41 | 'Department' column in the example above. 42 | 43 | Once defined, importers can be invoked to import a given data file. 44 | 45 | ```ruby 46 | EmployeeImporter.import('/path/to/file.xls') 47 | ``` 48 | 49 | The data file is expected to contain columns with titles corresponding to the 50 | columns declared. Any extra columns are ignored. Any errors while processing 51 | the data file does not interrupt the whole process. Instead, errors are 52 | notified via some callbacks defined in the importer (see below). 53 | 54 | ## Documentation 55 | 56 | For mote detailed information about the different aspects of importing data 57 | with `active_importer`, refer to the following sections in the [wiki](https://github.com/continuum/active_importer/wiki). 58 | 59 | ### Getting started 60 | 61 | * [Understanding how spreadsheets are parsed](https://github.com/continuum/active_importer/wiki/Understanding-how-spreadsheets-are-parsed) 62 | * [Mapping columns to attributes](https://github.com/continuum/active_importer/wiki/Mapping-columns-to-attributes) 63 | 64 | ### Diving in 65 | 66 | * [Custom data processing](https://github.com/continuum/active_importer/wiki/Custom-data-processing) 67 | * [Helper methods](https://github.com/continuum/active_importer/wiki/Helper-methods) 68 | * [File extension and supported formats](https://github.com/continuum/active_importer/wiki/File-extension-and-supported-formats) 69 | * [Passing custom parameters](https://github.com/continuum/active_importer/wiki/Custom-parameters) 70 | * [Events and callbacks](https://github.com/continuum/active_importer/wiki/Callbacks) 71 | * [Selecting the model instance to import into (Update instead of create)](https://github.com/continuum/active_importer/wiki/Update-instead-of-create) 72 | * [Error handling](https://github.com/continuum/active_importer/wiki/Error-handling) 73 | * [Selecting the sheet to get data from](https://github.com/continuum/active_importer/wiki/Selecting-the-sheet-to-work-with) 74 | * [Skipping rows](https://github.com/continuum/active_importer/wiki/Skipping-rows) 75 | 76 | ### Advanced features 77 | 78 | * [Aborting the import process](https://github.com/continuum/active_importer/wiki/Aborting-the-import-process) 79 | * [Transactional importers](https://github.com/continuum/active_importer/wiki/Transactional-importers) 80 | 81 | ## Contributing 82 | 83 | Contributions are welcome! Take a look at our [contributions guide][] for 84 | details. 85 | 86 | [contributions guide]: https://github.com/continuum/active_importer/wiki/Contributing 87 | -------------------------------------------------------------------------------- /lib/active_importer/base.rb: -------------------------------------------------------------------------------- 1 | require 'roo' 2 | 3 | module ActiveImporter 4 | class Base 5 | 6 | # 7 | # DSL and class variables 8 | # 9 | 10 | @abort_message = nil 11 | 12 | def abort!(message) 13 | @abort_message = message 14 | end 15 | 16 | def aborted? 17 | !!@abort_message 18 | end 19 | 20 | def self.imports(klass) 21 | @model_class = klass 22 | end 23 | 24 | def self.columns 25 | @columns ||= {} 26 | end 27 | 28 | def self.model_class 29 | @model_class 30 | end 31 | 32 | def model_class 33 | self.class.model_class 34 | end 35 | 36 | def self.sheet(index) 37 | @sheet_index = index 38 | end 39 | 40 | def self.fetch_model(&block) 41 | @fetch_model_block = block 42 | end 43 | 44 | def self.fetch_model_block 45 | @fetch_model_block 46 | end 47 | 48 | def self.skip_rows_if(&block) 49 | @skip_rows_block = block 50 | end 51 | 52 | def self.skip_rows_block 53 | @skip_rows_block 54 | end 55 | 56 | def self.column(title, field = nil, options = nil, &block) 57 | title = title.to_s.strip unless title.is_a?(Integer) 58 | if columns[title] 59 | raise "Duplicate importer column '#{title}'" 60 | end 61 | 62 | if field.is_a?(Hash) 63 | raise "Invalid column '#{title}': expected a single set of options" unless options.nil? 64 | options = field 65 | field = nil 66 | else 67 | options ||= {} 68 | end 69 | 70 | if field.nil? && block_given? 71 | raise "Invalid column '#{title}': must have a corresponding attribute, or it shouldn't have a block" 72 | end 73 | 74 | columns[title] = { 75 | field_name: field, 76 | transform: block, 77 | optional: !!options[:optional], 78 | } 79 | end 80 | 81 | def self.import(file, options = {}) 82 | new(file, options).import 83 | end 84 | 85 | # 86 | # Transactions 87 | # 88 | 89 | def self.transactional(flag = true) 90 | if flag 91 | raise "Model class does not support transactions" unless @model_class.respond_to?(:transaction) 92 | end 93 | @transactional = !!flag 94 | end 95 | 96 | def self.transactional? 97 | @transactional || false 98 | end 99 | 100 | def transactional? 101 | @transactional || self.class.transactional? 102 | end 103 | 104 | def transaction 105 | if transactional? 106 | model_class.transaction { yield } 107 | else 108 | yield 109 | end 110 | end 111 | 112 | private :transaction 113 | 114 | # 115 | # Callbacks 116 | # 117 | 118 | EVENTS = [ 119 | :row_success, 120 | :row_error, 121 | :row_processing, 122 | :row_skipped, 123 | :row_processed, 124 | :import_started, 125 | :import_finished, 126 | :import_failed, 127 | :import_aborted, 128 | ] 129 | 130 | def self.event_handlers 131 | @event_handlers ||= EVENTS.inject({}) { |hash, event| hash.merge({event => []}) } 132 | end 133 | 134 | def self.on(event, &block) 135 | raise "Unknown ActiveImporter event '#{event}'" unless EVENTS.include?(event) 136 | event_handlers[event] << block 137 | end 138 | 139 | def fire_event(event, param = nil) 140 | self.class.send(:fire_event, self, event, param) 141 | unless self.class == ActiveImporter::Base 142 | self.class.superclass.send(:fire_event, self, event, param) 143 | end 144 | end 145 | 146 | def self.fire_event(instance, event, param = nil) 147 | event_handlers[event].each do |block| 148 | instance.instance_exec(param, &block) 149 | end 150 | end 151 | 152 | private :fire_event 153 | 154 | class << self 155 | private :fire_event 156 | private :fetch_model_block 157 | end 158 | 159 | # 160 | # Implementation 161 | # 162 | 163 | attr_reader :header, :row, :model 164 | attr_reader :row_count, :row_index 165 | attr_reader :row_errors 166 | attr_reader :params 167 | 168 | def initialize(file, options = {}) 169 | @row_errors = [] 170 | @params = options.delete(:params) 171 | @transactional = options.fetch(:transactional, self.class.transactional?) 172 | 173 | raise "Importer is declared transactional at the class level" if !@transactional && self.class.transactional? 174 | 175 | @book = Roo::Spreadsheet.open(file, options) 176 | load_sheet 177 | load_header 178 | 179 | @data_row_indices = ((@header_index+1)..@book.last_row) 180 | @row_count = @data_row_indices.count 181 | rescue => e 182 | @book = @header = nil 183 | @row_count = 0 184 | @row_index = 1 185 | fire_event :import_failed, e 186 | raise 187 | end 188 | 189 | def fetch_model_block 190 | self.class.send(:fetch_model_block) 191 | end 192 | 193 | def fetch_model 194 | if fetch_model_block 195 | self.instance_exec(&fetch_model_block) 196 | else 197 | model_class.new 198 | end 199 | end 200 | 201 | def import 202 | transaction do 203 | return if @book.nil? 204 | fire_event :import_started 205 | @data_row_indices.each do |index| 206 | @row_index = index 207 | @row = row_to_hash @book.row(index) 208 | if skip_row? 209 | fire_event :row_skipped 210 | next 211 | end 212 | import_row 213 | if aborted? 214 | fire_event :import_aborted, @abort_message 215 | break 216 | end 217 | end 218 | end 219 | rescue => e 220 | fire_event :import_aborted, e.message 221 | raise 222 | ensure 223 | fire_event :import_finished 224 | end 225 | 226 | def row_processed_count 227 | row_index - @header_index 228 | rescue 229 | 0 230 | end 231 | 232 | def row_success_count 233 | row_processed_count - row_errors.count 234 | end 235 | 236 | def row_error_count 237 | row_errors.count 238 | end 239 | 240 | private 241 | 242 | def columns 243 | self.class.columns 244 | end 245 | 246 | def self.skip_row_blocks 247 | @skip_row_blocks ||= begin 248 | klass = self 249 | result = [] 250 | while klass < ActiveImporter::Base 251 | block = klass.skip_rows_block 252 | result << block if block 253 | klass = klass.superclass 254 | end 255 | result 256 | end 257 | end 258 | 259 | def skip_row? 260 | self.class.skip_row_blocks.any? { |block| self.instance_exec(&block) } 261 | end 262 | 263 | def load_sheet 264 | sheet_index = self.class.instance_variable_get(:@sheet_index) 265 | if sheet_index 266 | sheet_index = @book.sheets[sheet_index-1] if sheet_index.is_a?(Fixnum) 267 | @book.default_sheet = sheet_index.to_s 268 | end 269 | end 270 | 271 | def find_header_index 272 | required_column_keys = columns.keys.reject { |title| title.is_a?(Integer) || columns[title][:optional] } 273 | (1..@book.last_row).each do |index| 274 | row = @book.row(index).map { |cell| cell.to_s.strip } 275 | return index if required_column_keys.all? { |item| row.include?(item) } 276 | end 277 | return nil 278 | end 279 | 280 | def load_header 281 | @header_index = find_header_index 282 | if @header_index 283 | @header = @book.row(@header_index).map(&:to_s).map(&:strip) 284 | @header.map!.with_index{|label, i| label.empty? ? i : label } 285 | else 286 | raise 'Spreadsheet does not contain all the expected columns' 287 | end 288 | end 289 | 290 | def import_row 291 | begin 292 | @model = fetch_model 293 | build_model 294 | save_model unless aborted? 295 | rescue => e 296 | @row_errors << { row_index: row_index, error_message: e.message } 297 | fire_event :row_error, e 298 | raise if transactional? 299 | return false 300 | end 301 | fire_event :row_success 302 | true 303 | ensure 304 | fire_event :row_processed 305 | end 306 | 307 | def save_model 308 | model.save! if model.new_record? || model.changed? 309 | end 310 | 311 | def build_model 312 | row.each_pair do |key, value| 313 | column_def = columns[key] 314 | next if column_def.nil? || column_def[:field_name].nil? 315 | field_name = column_def[:field_name] 316 | transform = column_def[:transform] 317 | value = self.instance_exec(value, &transform) if transform 318 | model.send("#{field_name}=", value) 319 | end 320 | fire_event :row_processing 321 | end 322 | 323 | def row_to_hash(row) 324 | hash = {} 325 | row.each_with_index do |value, index| 326 | if columns[@header[index]] 327 | hash[@header[index]] = value 328 | end 329 | end 330 | hash 331 | end 332 | end 333 | end 334 | -------------------------------------------------------------------------------- /spec/active_importer/base_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe ActiveImporter::Base do 4 | let(:spreadsheet_data) do 5 | [ 6 | [' Name ', 'Birth Date', 'Department', 'Manager'], 7 | ['John Doe', '2013-10-25', 'IT'], 8 | ['Jane Doe', '2013-10-26', 'Sales'], 9 | ] 10 | end 11 | 12 | let(:spreadsheet_data_with_errors) do 13 | [ 14 | ['List of employees'], 15 | ['Name', 'Birth Date', 'Department', 'Manager'], 16 | ['John Doe', '2013-10-25', 'IT'], 17 | ['Invalid', '2013-10-24', 'Management'], 18 | ['Invalid', '2013-10-24', 'Accounting'], 19 | ['Jane Doe', '2013-10-26', 'Sales'], 20 | ] 21 | end 22 | 23 | let(:importer) { EmployeeImporter.new('/dummy/file') } 24 | 25 | before do 26 | allow(Roo::Spreadsheet).to receive(:open).at_least(:once).and_return Spreadsheet.new(spreadsheet_data) 27 | EmployeeImporter.instance_variable_set(:@fetch_model_block, nil) 28 | EmployeeImporter.instance_variable_set(:@sheet_index, nil) 29 | EmployeeImporter.transactional(false) 30 | end 31 | 32 | describe '.column' do 33 | it 'does not allow a column with block and no attribute' do 34 | expect { EmployeeImporter.column('Dummy') {} }.to raise_error 35 | end 36 | end 37 | 38 | it 'imports all data from the spreadsheet into the model' do 39 | expect { EmployeeImporter.import('/dummy/file') }.to change(Employee, :count).by(2) 40 | end 41 | 42 | it 'notifies when each row has been imported successfully' do 43 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 44 | expect(importer).not_to receive(:row_error) 45 | expect(importer).to receive(:row_success).twice 46 | EmployeeImporter.import('/dummy/file') 47 | end 48 | 49 | it 'notifies when the import process starts and finishes' do 50 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 51 | expect(importer).to receive(:import_started).once 52 | expect(importer).to receive(:import_finished).once 53 | expect(importer).to receive(:base_import_finished).once 54 | EmployeeImporter.import('/dummy/file') 55 | end 56 | 57 | it 'can receive custom parameters via the `params` option' do 58 | importer = EmployeeImporter.new('/dummy/file', params: 'anything') 59 | expect(importer.params).to eql('anything') 60 | end 61 | 62 | context do 63 | let(:spreadsheet_data) do 64 | [ 65 | [' Name ', 'Birth Date', 'Department', 'Unused', 'Manager'], 66 | ['Mary', '2013-10-25', 'IT', 'hello'], 67 | ['John', '2013-10-26', 'Sales', 'world'], 68 | ] 69 | end 70 | 71 | it 'processes optional columns when present' do 72 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 73 | expect { 74 | EmployeeImporter.import('/dummy/file') 75 | }.to change(Employee.where.not(unused_field: nil), :count).by(2) 76 | end 77 | end 78 | 79 | context do 80 | let(:spreadsheet_data) { spreadsheet_data_with_errors } 81 | 82 | before do 83 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 84 | EmployeeImporter.import('/dummy/file') 85 | end 86 | 87 | describe '.row_processed_count' do 88 | it 'reports the number of rows processed' do 89 | expect(importer.row_processed_count).to eq(4) 90 | end 91 | end 92 | 93 | describe '.row_success_count' do 94 | it 'reports the number of rows imported successfully' do 95 | expect(importer.row_success_count).to eq(2) 96 | end 97 | end 98 | 99 | describe '.row_error_count' do 100 | it 'reports the number of rows with errors' do 101 | expect(importer.row_error_count).to eq(2) 102 | end 103 | end 104 | end 105 | 106 | context 'when there are rows with errors' do 107 | let(:spreadsheet_data) { spreadsheet_data_with_errors } 108 | 109 | it 'does not import those rows' do 110 | expect { EmployeeImporter.import('/dummy/file') }.to change(Employee, :count).by(2) 111 | end 112 | 113 | it 'notifies about each error' do 114 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 115 | expect(importer).to receive(:row_error).twice 116 | expect(importer).to receive(:row_success).twice 117 | EmployeeImporter.import('/dummy/file') 118 | end 119 | 120 | it 'keeps track of each error' do 121 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 122 | expect { EmployeeImporter.import('/dummy/file') }.to change(importer.row_errors, :count).by(2) 123 | end 124 | 125 | it 'still notifies all rows as processed' do 126 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 127 | expect(importer).to receive(:row_processed).exactly(4).times 128 | EmployeeImporter.import('/dummy/file') 129 | end 130 | end 131 | 132 | context 'when the import fails' do 133 | let(:spreadsheet_data) do 134 | [ 135 | ['Name', 'Birth Date', 'Manager'], 136 | ['John Doe', '2013-10-25'], 137 | ['Jane Doe', '2013-10-26'], 138 | ] 139 | end 140 | 141 | it 'notifies the failure' do 142 | expect_any_instance_of(EmployeeImporter).to receive(:import_failed) 143 | expect { 144 | EmployeeImporter.import('/dummy/file') 145 | }.to raise_error 146 | end 147 | end 148 | 149 | context 'when header row is not the first one' do 150 | let(:spreadsheet_data) do 151 | [ 152 | [], 153 | ['List of employees', '', nil, 'Company Name'], 154 | ['Ordered by', 'Birth Date'], 155 | ['Name', 'Department', 'Birth Date', 'Manager'], 156 | ['John Doe', 'IT', '2013-10-25'], 157 | ['Jane Doe', 'Sales', '2013-10-26'], 158 | ] 159 | end 160 | 161 | it 'smartly skips any rows before the header' do 162 | expect { EmployeeImporter.import('/dummy/file') }.to change(Employee, :count).by(2) 163 | end 164 | 165 | it 'reports the number of processed rows correctly' do 166 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 167 | EmployeeImporter.import('/dummy/file') 168 | expect(importer.row_processed_count).to eq(2) 169 | end 170 | end 171 | 172 | context 'when header row is indented' do 173 | let(:spreadsheet_data) do 174 | [ 175 | ['', 'Name' , 'Department', 'Birth Date', 'Manager'], 176 | ['', 'John Doe', 'IT' , '2013-10-25' ], 177 | ] 178 | end 179 | 180 | it 'ignores empty columns' do 181 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 182 | EmployeeImporter.import('/dummy/file') 183 | expect(importer.row['']).to eq(nil) 184 | end 185 | end 186 | 187 | describe '.fetch_model' do 188 | it 'controls what model instance is loaded for each given row' do 189 | model = Employee.new 190 | EmployeeImporter.fetch_model { model } 191 | expect { EmployeeImporter.import('/dummy/file') }.to change(Employee, :count).by(1) 192 | end 193 | end 194 | 195 | describe 'row_processing event' do 196 | it 'allows the importer to modify the model for each row' do 197 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 198 | expect(importer).to receive(:row_processing).twice 199 | EmployeeImporter.import('/dummy/file') 200 | end 201 | end 202 | 203 | context 'when spreadsheet has multiple sheets' do 204 | let(:spreadsheet_data) do 205 | { 206 | "Employees" => [ 207 | [' Name ', 'Birth Date', 'Department', 'Manager'], 208 | ['John Doe', '2013-10-25', 'IT'], 209 | ['Jane Doe', '2013-10-26', 'Sales'], 210 | ], 211 | "Outstanding employees" => [ 212 | [' Name ', 'Birth Date', 'Department', 'Manager'], 213 | ['Jane Doe', '2013-10-26', 'Sales'], 214 | ], 215 | } 216 | end 217 | 218 | it 'uses the first sheet by default' do 219 | expect { EmployeeImporter.import('/dummy/file') }.to change(Employee, :count).by(2) 220 | end 221 | 222 | it 'uses another sheet if instructed to do so' do 223 | EmployeeImporter.sheet 1 224 | expect { EmployeeImporter.import('/dummy/file') }.to change(Employee, :count).by(2) 225 | EmployeeImporter.sheet "Outstanding employees" 226 | expect { EmployeeImporter.import('/dummy/file') }.to change(Employee, :count).by(1) 227 | end 228 | 229 | it 'fails if the specified sheet cannot be found' do 230 | expect_any_instance_of(EmployeeImporter).to receive(:import_failed) 231 | EmployeeImporter.sheet 5 232 | expect { 233 | EmployeeImporter.import('/dummy/file') 234 | }.to raise_error 235 | end 236 | end 237 | 238 | describe '#abort!' do 239 | let(:spreadsheet_data) do 240 | [ 241 | [' Name ', 'Birth Date', 'Department', 'Manager'], 242 | ['John Doe', '2013-10-25', 'IT'], 243 | ['Abort', '2013-10-25', 'IT'], 244 | ['Jane Doe', '2013-10-26', 'Sales'], 245 | ] 246 | end 247 | 248 | it 'causes the import process to abort without processing any more rows' do 249 | expect { EmployeeImporter.import('/dummy/file') }.to change(Employee, :count).by(1) 250 | end 251 | 252 | it 'does not report an error for the row where the abortion occured' do 253 | expect(importer).not_to receive(:row_error) 254 | EmployeeImporter.import('/dummy/file') 255 | end 256 | end 257 | 258 | describe '.skip_rows_if' do 259 | let(:spreadsheet_data) do 260 | [ 261 | [' Name ', 'Birth Date', 'Department', 'Manager'], 262 | ['Skip', '2013-10-25', 'IT'], 263 | ['John Doe', '2013-10-25', 'IT'], 264 | ['BaseSkip', '2013-10-25', 'IT'], 265 | ['Jane Doe', '2013-10-26', 'Sales'], 266 | ] 267 | end 268 | 269 | it 'allows the user to define conditions under which rows should be skipped' do 270 | expect { EmployeeImporter.import('/dummy/file') }.to change(Employee, :count).by(2) 271 | end 272 | 273 | it 'invokes event :row_skipped for each skipped row' do 274 | expect(EmployeeImporter).to receive(:new).once.and_return(importer) 275 | expect(importer).to receive(:row_skipped).twice 276 | EmployeeImporter.import('/dummy/file') 277 | end 278 | end 279 | 280 | describe '#initialize' do 281 | context "when invoked with option 'transactional: true'" do 282 | it 'declares the instance to be transactional even when the importer class is not' do 283 | EmployeeImporter.transactional(false) 284 | importer = EmployeeImporter.new('/dummy/file', transactional: true) 285 | expect(importer).to be_transactional 286 | end 287 | end 288 | 289 | context "when invoked with option 'transactional: false'" do 290 | it 'does not override the class-wide setting' do 291 | EmployeeImporter.transactional(true) 292 | expect_any_instance_of(EmployeeImporter).to receive(:import_failed) 293 | expect { 294 | EmployeeImporter.new('/dummy/file', transactional: false) 295 | }.to raise_error 296 | end 297 | end 298 | end 299 | 300 | describe '.transactional' do 301 | let(:spreadsheet_data) { spreadsheet_data_with_errors } 302 | 303 | before(:each) do 304 | allow(EmployeeImporter).to receive(:new).once.and_return(importer) 305 | end 306 | 307 | context 'when called with true as an argument' do 308 | before(:each) { EmployeeImporter.transactional(true) } 309 | 310 | it 'declares all importers of its kind to be transactional' do 311 | expect(EmployeeImporter).to be_transactional 312 | importer = EmployeeImporter.new('/dummy/file') 313 | expect(importer).to be_transactional 314 | end 315 | 316 | it 'runs the import process within a transaction' do 317 | expect { 318 | EmployeeImporter.import('/dummy/file') rescue nil 319 | }.not_to change(Employee, :count) 320 | end 321 | 322 | it 'exposes the exception that aborted the transaction' do 323 | expect { 324 | EmployeeImporter.import('/dummy/file') 325 | }.to raise_error 326 | end 327 | 328 | it 'still invokes the :row_error event' do 329 | expect(importer).to receive(:row_error) 330 | EmployeeImporter.import('/dummy/file') rescue nil 331 | end 332 | 333 | it 'still invokes the :import_finished event' do 334 | expect(importer).to receive(:import_finished) 335 | EmployeeImporter.import('/dummy/file') rescue nil 336 | end 337 | 338 | it 'invokes the :import_aborted event' do 339 | expect(importer).to receive(:import_aborted) 340 | EmployeeImporter.import('/dummy/file') rescue nil 341 | end 342 | end 343 | 344 | context 'when called with false as an argument' do 345 | it 'does not run the import process within a transactio' do 346 | EmployeeImporter.transactional(false) 347 | expect { 348 | EmployeeImporter.import('/dummy/file') 349 | }.to change(Employee, :count).by(2) 350 | end 351 | 352 | it 'declares all importers of its kind not to be transactional' do 353 | expect(EmployeeImporter).not_to be_transactional 354 | importer = EmployeeImporter.new('/dummy/file') 355 | expect(importer).not_to be_transactional 356 | end 357 | end 358 | end 359 | end 360 | --------------------------------------------------------------------------------