├── lib ├── pluck_in_batches │ ├── version.rb │ ├── extensions.rb │ └── iterator.rb └── pluck_in_batches.rb ├── gemfiles ├── activerecord_60.gemfile ├── activerecord_61.gemfile ├── activerecord_70.gemfile ├── activerecord_71.gemfile ├── activerecord_72.gemfile ├── activerecord_80.gemfile └── activerecord_head.gemfile ├── .gitignore ├── Rakefile ├── Gemfile ├── CHANGELOG.md ├── test ├── support │ ├── models.rb │ └── schema.rb ├── test_helper.rb └── pluck_in_batches_test.rb ├── pluck_in_batches.gemspec ├── .rubocop.yml ├── LICENSE.txt ├── .github └── workflows │ └── ci.yml ├── Gemfile.lock └── README.md /lib/pluck_in_batches/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PluckInBatches 4 | VERSION = "0.3.0" 5 | end 6 | -------------------------------------------------------------------------------- /gemfiles/activerecord_60.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | @ar_gem_requirement = "~> 6.0.0" 4 | 5 | eval_gemfile "../Gemfile" 6 | -------------------------------------------------------------------------------- /gemfiles/activerecord_61.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | @ar_gem_requirement = "~> 6.1.0" 4 | 5 | eval_gemfile "../Gemfile" 6 | -------------------------------------------------------------------------------- /gemfiles/activerecord_70.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | @ar_gem_requirement = "~> 7.0.0" 4 | 5 | eval_gemfile "../Gemfile" 6 | -------------------------------------------------------------------------------- /gemfiles/activerecord_71.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | @ar_gem_requirement = "~> 7.1.0" 4 | 5 | eval_gemfile "../Gemfile" 6 | -------------------------------------------------------------------------------- /gemfiles/activerecord_72.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | @ar_gem_requirement = "~> 7.2.0" 4 | 5 | eval_gemfile "../Gemfile" 6 | -------------------------------------------------------------------------------- /gemfiles/activerecord_80.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | @ar_gem_requirement = "~> 8.0.0.rc2" 4 | 5 | eval_gemfile "../Gemfile" 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | 10 | /debug.log* 11 | gemfiles/**.lock 12 | -------------------------------------------------------------------------------- /gemfiles/activerecord_head.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | @ar_gem_requirement = { github: "rails/rails", branch: "main" } 4 | 5 | eval_gemfile "../Gemfile" 6 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/gem_tasks" 4 | require "rake/testtask" 5 | require "rubocop/rake_task" 6 | 7 | Rake::TestTask.new(:test) do |t| 8 | t.libs << "test" 9 | t.libs << "lib" 10 | t.test_files = FileList["test/**/*_test.rb"] 11 | end 12 | 13 | RuboCop::RakeTask.new 14 | 15 | task default: [:rubocop, :test] 16 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | # Specify your gem's dependencies in pluck_in_batches.gemspec 6 | gemspec 7 | 8 | gem "rake", "~> 13.0" 9 | gem "minitest", "~> 5.0" 10 | gem "rubocop", "< 2" 11 | gem "rubocop-minitest" 12 | 13 | gem "sqlite3" 14 | 15 | if defined?(@ar_gem_requirement) 16 | gem "activerecord", @ar_gem_requirement 17 | else 18 | gem "activerecord" # latest 19 | end 20 | -------------------------------------------------------------------------------- /lib/pluck_in_batches.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "active_record" 4 | 5 | require_relative "pluck_in_batches/iterator" 6 | require_relative "pluck_in_batches/extensions" 7 | require_relative "pluck_in_batches/version" 8 | 9 | module PluckInBatches 10 | end 11 | 12 | ActiveSupport.on_load(:active_record) do 13 | extend(PluckInBatches::Extensions::ModelExtension) 14 | ActiveRecord::Relation.include(PluckInBatches::Extensions::RelationExtension) 15 | end 16 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## master (unreleased) 2 | 3 | ## 0.3.0 (2024-11-03) 4 | 5 | - Support plucking custom Arel columns 6 | 7 | ```ruby 8 | User.pluck_in_batches(:id, Arel.sql("json_extract(users.metadata, '$.rank')")) 9 | ``` 10 | 11 | ## 0.2.0 (2023-07-24) 12 | 13 | - Support specifying per cursor column ordering when batching 14 | 15 | ```ruby 16 | Book.pluck_in_batches(:title, cursor_columns: [:author_id, :version], order: [:asc, :desc]) 17 | ``` 18 | 19 | - Add `:of` as an alias for `:batch_size` option 20 | 21 | ## 0.1.0 (2023-05-16) 22 | 23 | - First release 24 | -------------------------------------------------------------------------------- /test/support/models.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class User < ActiveRecord::Base 4 | end 5 | 6 | class UserWithDefaultScope < ActiveRecord::Base 7 | self.table_name = :users 8 | default_scope { order(name: :desc) } 9 | end 10 | 11 | class SpecialUserWithDefaultScope < ActiveRecord::Base 12 | self.table_name = :users 13 | default_scope { where(id: [1, 5, 6]) } 14 | end 15 | 16 | class Subscriber < ActiveRecord::Base 17 | self.primary_key = :nick 18 | end 19 | 20 | class Product < ActiveRecord::Base 21 | self.primary_key = [:shop_id, :id] 22 | end 23 | 24 | class Package < ActiveRecord::Base 25 | end 26 | -------------------------------------------------------------------------------- /test/support/schema.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | ActiveRecord::Schema.define do 4 | create_table :users, force: true do |t| 5 | t.string :name 6 | t.json :metadata, null: false, default: {} 7 | end 8 | 9 | create_table :subscribers, id: false, primary_key: :nick, force: true do |t| 10 | t.string :nick 11 | t.string :name 12 | end 13 | 14 | create_table :products, primary_key: [:shop_id, :id], force: true do |t| 15 | t.integer :shop_id 16 | t.integer :id 17 | t.string :name 18 | end 19 | 20 | create_table :packages, id: :string, force: true do |t| 21 | t.integer :version 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /pluck_in_batches.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "lib/pluck_in_batches/version" 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "pluck_in_batches" 7 | spec.version = PluckInBatches::VERSION 8 | spec.authors = ["fatkodima"] 9 | spec.email = ["fatkodima123@gmail.com"] 10 | 11 | spec.summary = "A faster alternative to the custom use of `in_batches` with `pluck`." 12 | spec.homepage = "https://github.com/fatkodima/pluck_in_batches" 13 | spec.license = "MIT" 14 | spec.required_ruby_version = ">= 2.7.0" 15 | 16 | spec.metadata["homepage_uri"] = spec.homepage 17 | spec.metadata["source_code_uri"] = spec.homepage 18 | spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md" 19 | 20 | spec.files = Dir["*.{md,txt}", "lib/**/*"] 21 | spec.require_paths = ["lib"] 22 | 23 | spec.add_dependency "activerecord", ">= 6.0" 24 | end 25 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | require: rubocop-minitest 2 | 3 | AllCops: 4 | TargetRubyVersion: 2.7 5 | NewCops: enable 6 | SuggestExtensions: false 7 | 8 | Style/StringLiterals: 9 | EnforcedStyle: double_quotes 10 | 11 | Style/SymbolArray: 12 | EnforcedStyle: brackets 13 | 14 | Style/WordArray: 15 | EnforcedStyle: brackets 16 | 17 | Style/MultipleComparison: 18 | Enabled: false 19 | 20 | Style/Documentation: 21 | Enabled: false 22 | 23 | Style/NumericPredicate: 24 | Enabled: false 25 | 26 | Style/IfUnlessModifier: 27 | Enabled: false 28 | 29 | Layout/IndentationConsistency: 30 | EnforcedStyle: indented_internal_methods 31 | 32 | Layout/EmptyLinesAroundAccessModifier: 33 | EnforcedStyle: only_before 34 | 35 | Layout/LineLength: 36 | Enabled: false 37 | 38 | Lint/EmptyBlock: 39 | Exclude: 40 | - test/* 41 | 42 | Metrics: 43 | Enabled: false 44 | 45 | Bundler/OrderedGems: 46 | Enabled: false 47 | 48 | Gemspec/RequireMFA: 49 | Enabled: false 50 | 51 | Minitest/EmptyLineBeforeAssertionMethods: 52 | Enabled: false 53 | 54 | Minitest/MultipleAssertions: 55 | Enabled: false 56 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 fatkodima 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | # Run the linter first for rapid feedback if some trivial stylistic issues 6 | # slipped through the cracks. 7 | lint: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v3 11 | - uses: ruby/setup-ruby@v1 12 | with: 13 | ruby-version: 3.3 14 | bundler-cache: true 15 | - run: bundle exec rubocop 16 | 17 | test: 18 | needs: lint 19 | runs-on: ubuntu-latest 20 | strategy: 21 | matrix: 22 | include: 23 | - ruby-version: 2.7 24 | gemfile: activerecord_60.gemfile 25 | - ruby-version: 2.7 26 | gemfile: activerecord_61.gemfile 27 | - ruby-version: 2.7 28 | gemfile: activerecord_70.gemfile 29 | - ruby-version: 2.7 30 | gemfile: activerecord_71.gemfile 31 | - ruby-version: 3.3 32 | gemfile: activerecord_72.gemfile 33 | - ruby-version: 3.3 34 | gemfile: activerecord_80.gemfile 35 | 36 | # Test against latest versions just in case. 37 | - ruby-version: 3.3 38 | gemfile: activerecord_head.gemfile 39 | env: 40 | BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }} 41 | steps: 42 | - uses: actions/checkout@v3 43 | - uses: ruby/setup-ruby@v1 44 | with: 45 | ruby-version: ${{ matrix.ruby-version }} 46 | bundler-cache: true 47 | - run: bundle exec rake test 48 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | pluck_in_batches (0.3.0) 5 | activerecord (>= 6.0) 6 | 7 | GEM 8 | remote: https://rubygems.org/ 9 | specs: 10 | activemodel (7.2.2) 11 | activesupport (= 7.2.2) 12 | activerecord (7.2.2) 13 | activemodel (= 7.2.2) 14 | activesupport (= 7.2.2) 15 | timeout (>= 0.4.0) 16 | activesupport (7.2.2) 17 | base64 18 | benchmark (>= 0.3) 19 | bigdecimal 20 | concurrent-ruby (~> 1.0, >= 1.3.1) 21 | connection_pool (>= 2.2.5) 22 | drb 23 | i18n (>= 1.6, < 2) 24 | logger (>= 1.4.2) 25 | minitest (>= 5.1) 26 | securerandom (>= 0.3) 27 | tzinfo (~> 2.0, >= 2.0.5) 28 | ast (2.4.2) 29 | base64 (0.2.0) 30 | benchmark (0.3.0) 31 | bigdecimal (3.1.8) 32 | concurrent-ruby (1.3.4) 33 | connection_pool (2.4.1) 34 | drb (2.2.1) 35 | i18n (1.14.6) 36 | concurrent-ruby (~> 1.0) 37 | json (2.7.5) 38 | language_server-protocol (3.17.0.3) 39 | logger (1.6.1) 40 | minitest (5.25.1) 41 | parallel (1.26.3) 42 | parser (3.3.5.1) 43 | ast (~> 2.4.1) 44 | racc 45 | racc (1.8.1) 46 | rainbow (3.1.1) 47 | rake (13.2.1) 48 | regexp_parser (2.9.2) 49 | rubocop (1.68.0) 50 | json (~> 2.3) 51 | language_server-protocol (>= 3.17.0) 52 | parallel (~> 1.10) 53 | parser (>= 3.3.0.2) 54 | rainbow (>= 2.2.2, < 4.0) 55 | regexp_parser (>= 2.4, < 3.0) 56 | rubocop-ast (>= 1.32.2, < 2.0) 57 | ruby-progressbar (~> 1.7) 58 | unicode-display_width (>= 2.4.0, < 3.0) 59 | rubocop-ast (1.33.1) 60 | parser (>= 3.3.1.0) 61 | rubocop-minitest (0.36.0) 62 | rubocop (>= 1.61, < 2.0) 63 | rubocop-ast (>= 1.31.1, < 2.0) 64 | ruby-progressbar (1.13.0) 65 | securerandom (0.3.1) 66 | sqlite3 (2.2.0-x86_64-darwin) 67 | sqlite3 (2.2.0-x86_64-linux-gnu) 68 | timeout (0.4.1) 69 | tzinfo (2.0.6) 70 | concurrent-ruby (~> 1.0) 71 | unicode-display_width (2.6.0) 72 | 73 | PLATFORMS 74 | x86_64-darwin-20 75 | x86_64-darwin-21 76 | x86_64-linux 77 | 78 | DEPENDENCIES 79 | activerecord 80 | minitest (~> 5.0) 81 | pluck_in_batches! 82 | rake (~> 13.0) 83 | rubocop (< 2) 84 | rubocop-minitest 85 | sqlite3 86 | 87 | BUNDLED WITH 88 | 2.5.16 89 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH.unshift File.expand_path("../lib", __dir__) 4 | require "pluck_in_batches" 5 | 6 | require "sqlite3" 7 | require "securerandom" 8 | require "minitest/autorun" 9 | 10 | ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:") 11 | 12 | if ENV["VERBOSE"] 13 | ActiveRecord::Base.logger = ActiveSupport::Logger.new($stdout) 14 | else 15 | ActiveRecord::Base.logger = ActiveSupport::Logger.new("debug.log", 1, 100 * 1024 * 1024) # 100 mb 16 | ActiveRecord::Migration.verbose = false 17 | end 18 | 19 | require_relative "support/schema" 20 | require_relative "support/models" 21 | 22 | def prepare_database 23 | # Create users 24 | values = 20.times.map do |i| 25 | id = i + 1 26 | "(#{id}, 'User-#{id}', json('{\"rank\": #{i}}'))" 27 | end.join(", ") 28 | ActiveRecord::Base.connection.execute("INSERT INTO users (id, name, metadata) VALUES #{values}") 29 | 30 | # Create subscribers 31 | values = 10.times.map do |i| 32 | id = i + 1 33 | "('nick-#{id}', 'User-#{id}')" 34 | end.join(", ") 35 | ActiveRecord::Base.connection.execute("INSERT INTO subscribers (nick, name) VALUES #{values}") 36 | 37 | # Create products 38 | values = 20.times.map do |i| 39 | id = i + 1 40 | shop_id = rand(1..5) 41 | "(#{shop_id}, #{id}, '(#{shop_id}, #{id})')" 42 | end.join(", ") 43 | ActiveRecord::Base.connection.execute("INSERT INTO products (shop_id, id, name) VALUES #{values}") 44 | 45 | # Create packages 46 | values = 10.times.map do |i| 47 | id = SecureRandom.uuid 48 | version = i + 1 49 | "('#{id}', #{version})" 50 | end.join(", ") 51 | ActiveRecord::Base.connection.execute("INSERT INTO packages (id, version) VALUES #{values}") 52 | end 53 | 54 | prepare_database 55 | 56 | class TestCase < Minitest::Test 57 | alias assert_not_equal refute_equal 58 | 59 | def assert_nothing_raised 60 | yield.tap { assert(true) } # rubocop:disable Minitest/UselessAssertion 61 | rescue StandardError => e 62 | raise Minitest::UnexpectedError, e 63 | end 64 | 65 | def assert_queries(num, &block) 66 | ActiveRecord::Base.connection.materialize_transactions 67 | count = 0 68 | 69 | ActiveSupport::Notifications.subscribe("sql.active_record") do |_name, _start, _finish, _id, payload| 70 | count += 1 unless ["SCHEMA", "TRANSACTION"].include? payload[:name] 71 | end 72 | 73 | result = block.call 74 | assert_equal num, count, "#{count} instead of #{num} queries were executed." 75 | result 76 | end 77 | 78 | def assert_no_queries(&block) 79 | assert_queries(0, &block) 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/pluck_in_batches/extensions.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PluckInBatches 4 | module Extensions 5 | module ModelExtension 6 | delegate :pluck_each, :pluck_in_batches, to: :all 7 | end 8 | 9 | module RelationExtension 10 | # Yields each set of values corresponding to the specified columns that was found 11 | # by the passed options. If one column specified - returns its value, if an array of columns - 12 | # returns an array of values. 13 | # 14 | # See #pluck_in_batches for all the details. 15 | # 16 | def pluck_each(*columns, start: nil, finish: nil, of: 1000, batch_size: of, error_on_ignore: nil, order: :asc, cursor_column: primary_key, &block) 17 | iterator = Iterator.new(self) 18 | iterator.each(*columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order, &block) 19 | end 20 | 21 | # Yields each batch of values corresponding to the specified columns that was found 22 | # by the passed options as an array. 23 | # 24 | # User.where("age > 21").pluck_in_batches(:email) do |emails| 25 | # jobs = emails.map { |email| PartyReminderJob.new(email) } 26 | # ActiveJob.perform_all_later(jobs) 27 | # end 28 | # 29 | # If you do not provide a block to #pluck_in_batches, it will return an Enumerator 30 | # for chaining with other methods: 31 | # 32 | # User.pluck_in_batches(:name, :email).with_index do |group, index| 33 | # puts "Processing group ##{index}" 34 | # jobs = group.map { |name, email| PartyReminderJob.new(name, email) } 35 | # ActiveJob.perform_all_later(jobs) 36 | # end 37 | # 38 | # ==== Options 39 | # * :batch_size - Specifies the size of the batch. Defaults to 1000. 40 | # * :of - Same as +:batch_size+. 41 | # * :start - Specifies the primary key value to start from, inclusive of the value. 42 | # * :finish - Specifies the primary key value to end at, inclusive of the value. 43 | # * :error_on_ignore - Overrides the application config to specify if an error should be raised when 44 | # an order is present in the relation. 45 | # * :cursor_column - Specifies the column(s) on which the iteration should be done. 46 | # This column(s) should be orderable (e.g. an integer or string). Defaults to primary key. 47 | # * :order - Specifies the cursor column(s) order (can be +:asc+ or +:desc+ or an array consisting 48 | # of :asc or :desc). Defaults to +:asc+. 49 | # 50 | # class Book < ActiveRecord::Base 51 | # self.primary_key = [:author_id, :version] 52 | # end 53 | # 54 | # Book.pluck_in_batches(:title, order: [:asc, :desc]) 55 | # 56 | # In the above code, +author_id+ is sorted in ascending order and +version+ in descending order. 57 | # 58 | # Limits are honored, and if present there is no requirement for the batch 59 | # size: it can be less than, equal to, or greater than the limit. 60 | # 61 | # The options +start+ and +finish+ are especially useful if you want 62 | # multiple workers dealing with the same processing queue. You can make 63 | # worker 1 handle all the records between id 1 and 9999 and worker 2 64 | # handle from 10000 and beyond by setting the +:start+ and +:finish+ 65 | # option on each worker. 66 | # 67 | # # Let's process from record 10_000 on. 68 | # User.pluck_in_batches(:email, start: 10_000) do |emails| 69 | # jobs = emails.map { |email| PartyReminderJob.new(email) } 70 | # ActiveJob.perform_all_later(jobs) 71 | # end 72 | # 73 | # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to 74 | # ascending on the primary key ("id ASC"). 75 | # This also means that this method only works when the primary key is 76 | # orderable (e.g. an integer or string). 77 | # 78 | # NOTE: By its nature, batch processing is subject to race conditions if 79 | # other processes are modifying the database. 80 | # 81 | def pluck_in_batches(*columns, start: nil, finish: nil, of: 1000, batch_size: of, error_on_ignore: nil, cursor_column: primary_key, order: :asc, &block) 82 | iterator = Iterator.new(self) 83 | iterator.each_batch(*columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order, &block) 84 | end 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PluckInBatches 2 | 3 | [![Build Status](https://github.com/fatkodima/pluck_in_batches/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/fatkodima/pluck_in_batches/actions/workflows/ci.yml) 4 | 5 | ActiveRecord comes with `find_each` / `find_in_batches` / `in_batches` methods to batch process records from a database. 6 | ActiveRecord also has the `pluck` method which allows the selection of a set of fields without pulling 7 | the entire record into memory. 8 | 9 | This gem combines these ideas and provides `pluck_each` and `pluck_in_batches` methods to allow 10 | batch processing of plucked fields from the database. 11 | 12 | It performs half of the number of SQL queries, allocates up to half of the memory and is up to 2x faster 13 | (or more, depending on how far is your database from the application) than the available alternative: 14 | 15 | ```ruby 16 | # Before 17 | User.in_batches do |batch| # or .find_in_batches, or .select(:email).find_each etc 18 | emails = batch.pluck(:emails) 19 | # do something with emails 20 | end 21 | 22 | # Now, using this gem (up to 2x faster) 23 | User.pluck_in_batches(:email) do |emails| 24 | # do something with emails 25 | end 26 | ``` 27 | 28 | **Note**: You may also find [`sidekiq-iteration`](https://github.com/fatkodima/sidekiq-iteration) useful when iterating over large collections in Sidekiq jobs. 29 | 30 | ## Requirements 31 | 32 | - Ruby 2.7+ 33 | - ActiveRecord 6+ 34 | 35 | If you need support for older versions, [open an issue](https://github.com/fatkodima/pluck_in_batches/issues/new). 36 | 37 | ## Installation 38 | 39 | Add this line to your application's Gemfile: 40 | 41 | ```ruby 42 | gem 'pluck_in_batches' 43 | ``` 44 | 45 | And then execute: 46 | 47 | ```sh 48 | $ bundle 49 | ``` 50 | 51 | Or install it yourself as: 52 | 53 | ```sh 54 | $ gem install pluck_in_batches 55 | ``` 56 | 57 | ## Usage 58 | 59 | ### `pluck_each` 60 | 61 | Behaves similarly to `find_each` ActiveRecord's method, but yields each set of values corresponding 62 | to the specified columns. 63 | 64 | ```ruby 65 | # Single column 66 | User.where(active: true).pluck_each(:email) do |email| 67 | # do something with email 68 | end 69 | 70 | # Multiple columns 71 | User.where(active: true).pluck_each(:id, :email) do |id, email| 72 | # do something with id and email 73 | end 74 | ``` 75 | 76 | ### `pluck_in_batches` 77 | 78 | Behaves similarly to `in_batches` ActiveRecord's method, but yields each batch 79 | of values corresponding to the specified columns. 80 | 81 | ```ruby 82 | # Single column 83 | User.where("age > 21").pluck_in_batches(:email) do |emails| 84 | jobs = emails.map { |email| PartyReminderJob.new(email) } 85 | ActiveJob.perform_all_later(jobs) 86 | end 87 | 88 | # Multiple columns 89 | User.pluck_in_batches(:name, :email).with_index do |group, index| 90 | puts "Processing group ##{index}" 91 | jobs = group.map { |name, email| PartyReminderJob.new(name, email) } 92 | ActiveJob.perform_all_later(jobs) 93 | end 94 | 95 | # Custom arel column 96 | User.pluck_in_batches(:id, Arel.sql("json_extract(users.metadata, '$.rank')")).with_index do |group, index| 97 | # ... 98 | end 99 | ``` 100 | 101 | Both methods support the following configuration options: 102 | 103 | * `:batch_size` - Specifies the size of the batch. Defaults to 1000. 104 | Also aliased as `:of`. 105 | * `:start` - Specifies the primary key value to start from, inclusive of the value. 106 | * `:finish` - Specifies the primary key value to end at, inclusive of the value. 107 | * `:error_on_ignore` - Overrides the application config to specify if an error should be raised when 108 | an order is present in the relation. 109 | * `:cursor_column` - Specifies the column(s) on which the iteration should be done. 110 | This column(s) should be orderable (e.g. an integer or string). Defaults to primary key. 111 | * `:order` - Specifies the primary key order (can be `:asc` or `:desc` or 112 | an array consisting of :asc or :desc). Defaults to `:asc`. 113 | 114 | ## Development 115 | 116 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org). 117 | 118 | ## Contributing 119 | 120 | Bug reports and pull requests are welcome on GitHub at https://github.com/fatkodima/pluck_in_batches. 121 | 122 | ## License 123 | 124 | The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). 125 | -------------------------------------------------------------------------------- /lib/pluck_in_batches/iterator.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PluckInBatches 4 | class Iterator # :nodoc: 5 | VALID_ORDERS = [:asc, :desc].freeze 6 | DEFAULT_ORDER = :asc 7 | 8 | def initialize(relation) 9 | @relation = relation 10 | @klass = relation.klass 11 | end 12 | 13 | def each(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: @relation.primary_key, order: DEFAULT_ORDER, &block) 14 | if columns.empty? 15 | raise ArgumentError, "Call `pluck_each' with at least one column." 16 | end 17 | 18 | if block_given? 19 | each_batch(*columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order) do |batch| 20 | batch.each(&block) 21 | end 22 | else 23 | enum_for(__callee__, *columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order) do 24 | apply_limits(@relation, start, finish, build_batch_orders(order)).size 25 | end 26 | end 27 | end 28 | 29 | def each_batch(*columns, start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor_column: @relation.primary_key, order: DEFAULT_ORDER) 30 | if columns.empty? 31 | raise ArgumentError, "Call `pluck_in_batches' with at least one column." 32 | end 33 | 34 | unless Array(order).all? { |ord| VALID_ORDERS.include?(ord) } 35 | raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}" 36 | end 37 | 38 | pluck_columns = columns.map do |column| 39 | if Arel.arel_node?(column) 40 | column 41 | else 42 | column.to_s 43 | end 44 | end 45 | 46 | cursor_columns = Array(cursor_column).map(&:to_s) 47 | cursor_column_indexes = cursor_column_indexes(pluck_columns, cursor_columns) 48 | missing_cursor_columns = cursor_column_indexes.count(&:nil?) 49 | cursor_column_indexes.each_with_index do |column_index, index| 50 | unless column_index 51 | cursor_column_indexes[index] = pluck_columns.size 52 | pluck_columns << cursor_columns[index] 53 | end 54 | end 55 | 56 | relation = @relation 57 | batch_orders = build_batch_orders(cursor_columns, order) 58 | 59 | unless block_given? 60 | return to_enum(__callee__, *columns, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor_column: cursor_column, order: order) do 61 | total = apply_limits(relation, cursor_columns, start, finish, batch_orders).size 62 | (total - 1).div(batch_size) + 1 63 | end 64 | end 65 | 66 | if relation.arel.orders.present? 67 | act_on_ignored_order(error_on_ignore) 68 | end 69 | 70 | batch_limit = batch_size 71 | if relation.limit_value 72 | remaining = relation.limit_value 73 | batch_limit = remaining if remaining < batch_limit 74 | end 75 | 76 | relation = relation.reorder(batch_orders.to_h).limit(batch_limit) 77 | relation = apply_limits(relation, cursor_columns, start, finish, batch_orders) 78 | relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching 79 | batch_relation = relation 80 | 81 | loop do 82 | batch = batch_relation.pluck(*pluck_columns) 83 | break if batch.empty? 84 | 85 | cursor_column_offsets = 86 | if pluck_columns.size == 1 87 | Array(batch.last) 88 | else 89 | cursor_column_indexes.map.with_index do |column_index, index| 90 | batch.last[column_index || (batch.last.size - cursor_column_indexes.size + index)] 91 | end 92 | end 93 | 94 | missing_cursor_columns.times { batch.each(&:pop) } 95 | batch.flatten!(1) if columns.size == 1 96 | 97 | yield batch 98 | 99 | break if batch.length < batch_limit 100 | 101 | if @relation.limit_value 102 | remaining -= batch.length 103 | 104 | if remaining == 0 105 | # Saves a useless iteration when the limit is a multiple of the 106 | # batch size. 107 | break 108 | elsif remaining < batch_limit 109 | relation = relation.limit(remaining) 110 | end 111 | end 112 | 113 | _last_column, last_order = batch_orders.last 114 | operators = batch_orders.map do |_column, order| # rubocop:disable Lint/ShadowingOuterLocalVariable 115 | order == :desc ? :lteq : :gteq 116 | end 117 | operators[-1] = (last_order == :desc ? :lt : :gt) 118 | 119 | batch_relation = batch_condition(relation, cursor_columns, cursor_column_offsets, operators) 120 | end 121 | end 122 | 123 | private 124 | def cursor_column_indexes(columns, cursor_column) 125 | cursor_column.map do |column| 126 | columns.index(column) || 127 | columns.index("#{@klass.table_name}.#{column}") || 128 | columns.index("#{@klass.quoted_table_name}.#{@klass.connection.quote_column_name(column)}") 129 | end 130 | end 131 | 132 | def act_on_ignored_order(error_on_ignore) 133 | raise_error = 134 | if error_on_ignore.nil? 135 | if ar_version >= 7.0 136 | ActiveRecord.error_on_ignored_order 137 | else 138 | @klass.error_on_ignored_order 139 | end 140 | else 141 | error_on_ignore 142 | end 143 | 144 | message = "Scoped order is ignored, it's forced to be batch order." 145 | 146 | if raise_error 147 | raise ArgumentError, message 148 | elsif (logger = ActiveRecord::Base.logger) 149 | logger.warn(message) 150 | end 151 | end 152 | 153 | def apply_limits(relation, columns, start, finish, batch_orders) 154 | relation = apply_start_limit(relation, columns, start, batch_orders) if start 155 | relation = apply_finish_limit(relation, columns, finish, batch_orders) if finish 156 | relation 157 | end 158 | 159 | def apply_start_limit(relation, columns, start, batch_orders) 160 | operators = batch_orders.map do |_column, order| 161 | order == :desc ? :lteq : :gteq 162 | end 163 | batch_condition(relation, columns, start, operators) 164 | end 165 | 166 | def apply_finish_limit(relation, columns, finish, batch_orders) 167 | operators = batch_orders.map do |_column, order| 168 | order == :desc ? :gteq : :lteq 169 | end 170 | batch_condition(relation, columns, finish, operators) 171 | end 172 | 173 | def batch_condition(relation, columns, values, operators) 174 | cursor_positions = Array(columns).zip(Array(values), operators) 175 | 176 | first_clause_column, first_clause_value, operator = cursor_positions.pop 177 | where_clause = build_attribute_predicate(first_clause_column, first_clause_value, operator) 178 | 179 | cursor_positions.reverse_each do |column_name, value, operator| # rubocop:disable Lint/ShadowingOuterLocalVariable 180 | where_clause = build_attribute_predicate(column_name, value, operator == :lteq ? :lt : :gt).or( 181 | build_attribute_predicate(column_name, value, :eq).and(where_clause) 182 | ) 183 | end 184 | 185 | relation.where(where_clause) 186 | end 187 | 188 | def build_attribute_predicate(column, value, operator) 189 | @relation.bind_attribute(column, value) { |attr, bind| attr.public_send(operator, bind) } 190 | end 191 | 192 | def build_batch_orders(cursor_columns, order) 193 | cursor_columns.zip(Array(order)).map do |column, ord| 194 | [column, ord || DEFAULT_ORDER] 195 | end 196 | end 197 | 198 | def ar_version 199 | ActiveRecord.version.to_s.to_f 200 | end 201 | end 202 | end 203 | -------------------------------------------------------------------------------- /test/pluck_in_batches_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class PluckInBatchesTest < TestCase 6 | def test_pluck_each_requires_columns 7 | error = assert_raises(ArgumentError) do 8 | User.pluck_each 9 | end 10 | assert_match("Call `pluck_each' with at least one column.", error.message) 11 | end 12 | 13 | def test_pluck_each_should_return_an_enumerator_if_no_block_is_present 14 | ids = User.order(:id).ids 15 | assert_queries(1) do 16 | User.pluck_each(:id, batch_size: 100_000).with_index do |id, index| 17 | assert_equal ids[index], id 18 | assert_kind_of Integer, index 19 | end 20 | end 21 | end 22 | 23 | def test_pluck_each_should_return_values 24 | ids_and_names = User.order(:id).pluck(:id, :name) 25 | index = 0 26 | assert_queries(User.count + 1) do 27 | User.pluck_each(:id, :name, batch_size: 1) do |values| 28 | assert_equal ids_and_names[index], values 29 | index += 1 30 | end 31 | end 32 | end 33 | 34 | def test_pluck_in_batches_requires_columns 35 | error = assert_raises(ArgumentError) do 36 | User.pluck_in_batches 37 | end 38 | assert_match(error.message, "Call `pluck_in_batches' with at least one column.") 39 | end 40 | 41 | def test_pluck_in_batches_should_return_batches 42 | ids = User.order(:id).ids 43 | assert_queries(User.count + 1) do 44 | User.pluck_in_batches(:id, batch_size: 1).with_index do |batch, index| 45 | assert_kind_of Array, batch 46 | assert_equal ids[index], batch.first 47 | end 48 | end 49 | end 50 | 51 | def test_pluck_in_batches_should_return_batches_for_arel_columns 52 | ids_and_ranks = User.order(:id).pluck(:id, Arel.sql("json_extract(users.metadata, '$.rank')")) 53 | 54 | assert_queries(User.count + 1) do 55 | User.pluck_in_batches(:id, Arel.sql("json_extract(users.metadata, '$.rank')"), batch_size: 1).with_index do |batch, index| 56 | assert_kind_of Array, batch 57 | assert_equal ids_and_ranks[index], batch.first 58 | end 59 | end 60 | end 61 | 62 | def pluck_in_batches_desc_order 63 | ids = User.order(id: :desc).ids 64 | assert_queries(User.count + 1) do 65 | User.pluck_in_batches(:id, batch_size: 1, order: :desc).with_index do |batch, index| 66 | assert_kind_of Array, batch 67 | assert_equal ids[index], batch.first 68 | end 69 | end 70 | end 71 | 72 | def test_pluck_in_batches_should_start_from_the_start_option 73 | assert_queries(User.count) do 74 | User.pluck_in_batches(:id, batch_size: 1, start: 2) do |batch| 75 | assert_kind_of Array, batch 76 | assert_kind_of Integer, batch.first 77 | end 78 | end 79 | end 80 | 81 | def test_pluck_in_batches_should_end_at_the_finish_option 82 | assert_queries(6) do 83 | User.pluck_in_batches(:id, batch_size: 1, finish: 5) do |batch| 84 | assert_kind_of Array, batch 85 | assert_kind_of Integer, batch.first 86 | end 87 | end 88 | end 89 | 90 | def test_pluck_in_batches_multiple_columns 91 | ids_and_names = User.order(:id).pluck(:id, :name) 92 | assert_queries(User.count + 1) do 93 | User.pluck_in_batches(:id, :name, batch_size: 1).with_index do |batch, index| 94 | assert_kind_of Array, batch 95 | assert_kind_of Array, batch.first 96 | assert_equal ids_and_names[index], batch.first 97 | end 98 | end 99 | end 100 | 101 | def test_pluck_in_batches_id_is_missing 102 | names = User.order(:id).pluck(:name) 103 | assert_queries(User.count + 1) do 104 | User.pluck_in_batches(:name, batch_size: 1).with_index do |batch, index| 105 | assert_kind_of Array, batch 106 | assert_equal names[index], batch.first 107 | end 108 | end 109 | end 110 | 111 | def test_pluck_in_batches_fully_qualified_id_is_present 112 | ids_and_names = User.order(:id).pluck(:id, :name) 113 | assert_queries(User.count + 1) do 114 | User.pluck_in_batches("users.id", :name, batch_size: 1).with_index do |batch, index| 115 | assert_kind_of Array, batch 116 | assert_kind_of Array, batch.first 117 | assert_equal ids_and_names[index], batch.first 118 | end 119 | end 120 | end 121 | 122 | def test_pluck_in_batches_shouldnt_execute_query_unless_needed 123 | count = User.count 124 | assert_queries(2) do 125 | User.pluck_in_batches(:id, batch_size: count) { |batch| assert_kind_of Array, batch } 126 | end 127 | 128 | assert_queries(1) do 129 | User.pluck_in_batches(:id, batch_size: count + 1) { |batch| assert_kind_of Array, batch } 130 | end 131 | end 132 | 133 | def test_pluck_in_batches_should_ignore_the_order_default_scope 134 | # First user with name scope 135 | first_user = UserWithDefaultScope.first 136 | ids = [] 137 | UserWithDefaultScope.pluck_in_batches(:id) do |batch| 138 | ids.concat(batch) 139 | end 140 | # ids.first will be ordered using id only. Name order scope should not apply here 141 | assert_not_equal first_user.id, ids.first 142 | assert_equal User.first.id, ids.first 143 | end 144 | 145 | def test_pluck_in_batches_should_error_on_ignore_the_order 146 | error = assert_raises(ArgumentError) do 147 | UserWithDefaultScope.pluck_in_batches(:id, error_on_ignore: true) {} 148 | end 149 | assert_match("Scoped order is ignored, it's forced to be batch order.", error.message) 150 | end 151 | 152 | def test_pluck_in_batches_should_not_error_if_config_overridden 153 | with_error_on_ignored_order(UserWithDefaultScope, true) do 154 | assert_nothing_raised do 155 | UserWithDefaultScope.pluck_in_batches(:id, error_on_ignore: false) {} 156 | end 157 | end 158 | end 159 | 160 | def test_pluck_in_batches_should_error_on_config_specified_to_error 161 | with_error_on_ignored_order(UserWithDefaultScope, true) do 162 | error = assert_raises(ArgumentError) do 163 | UserWithDefaultScope.pluck_in_batches(:id) {} 164 | end 165 | assert_match("Scoped order is ignored, it's forced to be batch order.", error.message) 166 | end 167 | end 168 | 169 | def test_pluck_in_batches_should_not_error_by_default 170 | assert_nothing_raised do 171 | UserWithDefaultScope.pluck_in_batches(:id) {} 172 | end 173 | end 174 | 175 | def test_pluck_in_batches_should_not_ignore_the_default_scope_if_it_is_other_than_order 176 | default_scope = SpecialUserWithDefaultScope.all 177 | ids = [] 178 | SpecialUserWithDefaultScope.pluck_in_batches(:id) do |batch| 179 | ids.concat(batch) 180 | end 181 | assert_equal default_scope.pluck(:id).sort, ids.sort 182 | end 183 | 184 | def test_pluck_in_batches_should_use_any_column_as_primary_key 185 | nick_order_subscribers = Subscriber.order(nick: :asc) 186 | start_nick = nick_order_subscribers.second.nick 187 | 188 | names = [] 189 | Subscriber.pluck_in_batches(:name, batch_size: 1, start: start_nick) do |batch| 190 | names.concat(batch) 191 | end 192 | 193 | assert_equal nick_order_subscribers[1..].map(&:name), names 194 | end 195 | 196 | def test_pluck_in_batches_should_return_an_enumerator 197 | enum = nil 198 | assert_no_queries do 199 | enum = User.pluck_in_batches(:id, batch_size: 1) 200 | end 201 | assert_queries(4) do 202 | enum.first(4) do |batch| 203 | assert_kind_of Array, batch 204 | assert_kind_of Integer, batch.first 205 | end 206 | end 207 | end 208 | 209 | def test_pluck_in_batches_should_honor_limit_if_passed_a_block 210 | limit = User.count - 1 211 | total = 0 212 | 213 | User.limit(limit).pluck_in_batches(:id) do |batch| 214 | total += batch.size 215 | end 216 | 217 | assert_equal limit, total 218 | end 219 | 220 | def test_pluck_in_batches_should_honor_limit_if_no_block_is_passed 221 | limit = User.count - 1 222 | total = 0 223 | 224 | User.limit(limit).pluck_in_batches(:id).each do |batch| 225 | total += batch.size 226 | end 227 | 228 | assert_equal limit, total 229 | end 230 | 231 | def test_pluck_in_batches_should_return_a_sized_enumerator 232 | assert_equal 20, User.pluck_in_batches(:id, batch_size: 1).size 233 | assert_equal 10, User.pluck_in_batches(:id, batch_size: 2).size 234 | assert_equal 9, User.pluck_in_batches(:id, batch_size: 2, start: 4).size 235 | assert_equal 7, User.pluck_in_batches(:id, batch_size: 3).size 236 | assert_equal 1, User.pluck_in_batches(:id, batch_size: 10_000).size 237 | end 238 | 239 | module CompositePrimaryKeys 240 | def test_pluck_in_batches_should_iterate_over_composite_primary_key 241 | skip if ar_version < 7.1 242 | 243 | ids = Product.order(:shop_id, :id).ids 244 | Product.pluck_in_batches(:shop_id, :id, batch_size: 1).with_index do |batch, index| 245 | assert_kind_of Array, batch 246 | assert_equal ids[index], batch.first 247 | end 248 | end 249 | 250 | def test_pluck_in_batches_over_composite_primary_key_when_one_column_is_missing 251 | skip if ar_version < 7.1 252 | 253 | ids_and_names = Product.order(:shop_id, :id).pluck(:id, :name) 254 | Product.pluck_in_batches(:id, :name, batch_size: 1).with_index do |batch, index| 255 | assert_kind_of Array, batch 256 | assert_equal ids_and_names[index], batch.first 257 | end 258 | end 259 | 260 | def test_pluck_in_batches_over_composite_primary_key_should_start_from_the_start_option 261 | skip if ar_version < 7.1 262 | 263 | product = Product.second 264 | batch = Product.pluck_in_batches(:name, batch_size: 1, start: product.id).first 265 | assert_equal product.name, batch.first 266 | end 267 | 268 | def test_pluck_in_batches_over_composite_primary_key_should_end_at_the_finish_option 269 | skip if ar_version < 7.1 270 | 271 | product = Product.second_to_last 272 | batch = Product.pluck_in_batches(:name, batch_size: 1, finish: product.id).reverse_each.first 273 | assert_equal product.name, batch.first 274 | end 275 | 276 | def test_pluck_in_batches_should_iterate_over_composite_primary_key_using_multiple_column_ordering 277 | skip if ar_version < 7.1 278 | 279 | ids = Product.order(shop_id: :asc, id: :desc).ids 280 | index = 0 281 | Product.pluck_in_batches(:shop_id, :id, batch_size: 1, order: [:asc, :desc]) do |batch| 282 | assert_kind_of Array, batch 283 | assert_equal ids[index], batch.first 284 | index += 1 285 | end 286 | assert_equal ids.size, index 287 | end 288 | 289 | def test_pluck_in_batches_over_composite_primary_key_using_multiple_column_ordering_should_start_from_the_start_option 290 | skip if ar_version < 7.1 291 | 292 | product = Product.order(shop_id: :asc, id: :desc).second 293 | batch = Product.pluck_in_batches(:name, batch_size: 1, start: product.id, order: [:asc, :desc]).first 294 | assert_equal product.name, batch.first 295 | end 296 | 297 | def test_pluck_in_batches_over_composite_primary_key_using_multiple_column_ordering_should_end_at_the_finish_option 298 | skip if ar_version < 7.1 299 | 300 | product = Product.order(shop_id: :asc, id: :desc).second_to_last 301 | batch = Product.pluck_in_batches(:name, batch_size: 1, finish: product.id, order: [:asc, :desc]).to_a.last 302 | assert_equal product.name, batch.first 303 | end 304 | end 305 | include CompositePrimaryKeys 306 | 307 | def test_pluck_each_should_iterate_over_custom_cursor_column 308 | ids = Package.order(:version).ids 309 | Package.pluck_each(:id, batch_size: 1, cursor_column: :version).with_index do |id, index| 310 | assert_equal ids[index], id 311 | end 312 | end 313 | 314 | def test_pluck_in_batches_should_iterate_over_custom_cursor_column 315 | ids = Package.order(:version).ids 316 | Package.pluck_in_batches(:id, batch_size: 1, cursor_column: :version).with_index do |batch, index| 317 | assert_equal ids[index], batch.first 318 | end 319 | end 320 | 321 | private 322 | def with_error_on_ignored_order(klass, value) 323 | if ar_version >= 7.0 324 | prev = ActiveRecord.error_on_ignored_order 325 | ActiveRecord.error_on_ignored_order = value 326 | else 327 | prev = klass.error_on_ignored_order 328 | klass.error_on_ignored_order = value 329 | end 330 | yield 331 | ensure 332 | if ar_version >= 7.0 333 | ActiveRecord.error_on_ignored_order = prev 334 | else 335 | klass.error_on_ignored_order = prev 336 | end 337 | end 338 | 339 | def ar_version 340 | ActiveRecord.version.to_s.to_f 341 | end 342 | end 343 | --------------------------------------------------------------------------------