├── .github ├── CODEOWNERS └── workflows │ ├── ci.yml │ └── gem-push.yml ├── .gitignore ├── .rspec ├── Gemfile ├── Gemfile.6.0 ├── Gemfile.6.0.lock ├── Gemfile.6.1 ├── Gemfile.6.1.lock ├── LICENSE ├── README.md ├── Rakefile ├── jit_preloader.gemspec ├── lib ├── jit_preloader.rb └── jit_preloader │ ├── active_record │ ├── associations │ │ ├── collection_association.rb │ │ ├── preloader │ │ │ ├── ar6_association.rb │ │ │ ├── ar7_association.rb │ │ │ ├── ar7_branch.rb │ │ │ ├── collection_association.rb │ │ │ └── singular_association.rb │ │ └── singular_association.rb │ ├── base.rb │ └── relation.rb │ ├── preloader.rb │ └── version.rb └── spec ├── lib └── jit_preloader │ ├── active_record │ └── base_spec.rb │ └── preloader_spec.rb ├── spec_helper.rb └── support ├── database.rb └── models.rb /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Own any files in the .github/workflows directory and any of its 2 | # subdirectories. 3 | /.github/workflows/ @clio/application-security @clio/penguins 4 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | timeout-minutes: 5 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | gemfile: 17 | - Gemfile 18 | - Gemfile.6.1 19 | env: 20 | BUNDLE_GEMFILE: ${{ matrix.gemfile }} 21 | steps: 22 | - uses: actions/checkout@v4 23 | - name: Set up Ruby ${{ matrix.ruby-version }} 24 | uses: ruby/setup-ruby@v1 25 | with: 26 | ruby-version: 3.1 27 | - name: Install dependencies 28 | run: bundle install 29 | - name: Run tests 30 | run: 31 | bundle exec rspec -------------------------------------------------------------------------------- /.github/workflows/gem-push.yml: -------------------------------------------------------------------------------- 1 | name: Ruby Gem 2 | 3 | on: 4 | release: 5 | types: [ published ] 6 | 7 | jobs: 8 | build: 9 | name: Build + Publish 10 | runs-on: ubuntu-latest 11 | permissions: 12 | contents: read 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Set up Ruby 3.1 17 | uses: ruby/setup-ruby@v1 18 | with: 19 | ruby-version: 3.1 20 | 21 | - name: Publish to RubyGems 22 | env: 23 | GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}" 24 | run: | 25 | mkdir -p $HOME/.gem 26 | touch $HOME/.gem/credentials 27 | chmod 0600 $HOME/.gem/credentials 28 | printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials 29 | gem build *.gemspec 30 | gem push *.gem 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .byebug_history 4 | /.config 5 | /coverage/ 6 | /InstalledFiles 7 | /pkg/ 8 | /spec/reports/ 9 | /spec/examples.txt 10 | /test/tmp/ 11 | /test/version_tmp/ 12 | /tmp/ 13 | .idea/* 14 | 15 | # Used by dotenv library to load environment variables. 16 | # .env 17 | 18 | ## Specific to RubyMotion: 19 | .dat* 20 | .repl_history 21 | build/ 22 | *.bridgesupport 23 | build-iPhoneOS/ 24 | build-iPhoneSimulator/ 25 | 26 | ## Specific to RubyMotion (use of CocoaPods): 27 | # 28 | # We recommend against adding the Pods directory to your .gitignore. However 29 | # you should judge for yourself, the pros and cons are mentioned at: 30 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 31 | # 32 | # vendor/Pods/ 33 | 34 | ## Documentation cache and generated files: 35 | /.yardoc/ 36 | /_yardoc/ 37 | /doc/ 38 | /rdoc/ 39 | 40 | ## Environment normalization: 41 | /.bundle/ 42 | /vendor/bundle 43 | /lib/bundler/man/ 44 | 45 | # for a library or gem, you might want to ignore these files since the code is 46 | # intended to run in multiple environments; otherwise, check them in: 47 | Gemfile.lock 48 | # .ruby-version 49 | # .ruby-gemset 50 | 51 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 52 | .rvmrc 53 | .ruby-version -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --require spec_helper 3 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem "activerecord", ">=7" 4 | gem "sqlite3", "~> 1.4" 5 | # Specify your gem's dependencies in jit_preloader.gemspec 6 | gemspec 7 | -------------------------------------------------------------------------------- /Gemfile.6.0: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem "activerecord", "~>6.0.0" 4 | 5 | # Specify your gem's dependencies in jit_preloader.gemspec 6 | gemspec 7 | -------------------------------------------------------------------------------- /Gemfile.6.0.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | jit_preloader (1.0.3) 5 | activerecord (>= 5.2, < 7) 6 | activesupport 7 | 8 | GEM 9 | remote: https://rubygems.org/ 10 | specs: 11 | activemodel (6.0.4.1) 12 | activesupport (= 6.0.4.1) 13 | activerecord (6.0.4.1) 14 | activemodel (= 6.0.4.1) 15 | activesupport (= 6.0.4.1) 16 | activesupport (6.0.4.1) 17 | concurrent-ruby (~> 1.0, >= 1.0.2) 18 | i18n (>= 0.7, < 2) 19 | minitest (~> 5.1) 20 | tzinfo (~> 1.1) 21 | zeitwerk (~> 2.2, >= 2.2.2) 22 | byebug (11.1.3) 23 | concurrent-ruby (1.1.9) 24 | database_cleaner (2.0.1) 25 | database_cleaner-active_record (~> 2.0.0) 26 | database_cleaner-active_record (2.0.1) 27 | activerecord (>= 5.a) 28 | database_cleaner-core (~> 2.0.0) 29 | database_cleaner-core (2.0.1) 30 | db-query-matchers (0.10.0) 31 | activesupport (>= 4.0, < 7) 32 | rspec (~> 3.0) 33 | diff-lcs (1.4.4) 34 | i18n (1.8.10) 35 | concurrent-ruby (~> 1.0) 36 | minitest (5.14.4) 37 | rake (13.0.6) 38 | rspec (3.10.0) 39 | rspec-core (~> 3.10.0) 40 | rspec-expectations (~> 3.10.0) 41 | rspec-mocks (~> 3.10.0) 42 | rspec-core (3.10.1) 43 | rspec-support (~> 3.10.0) 44 | rspec-expectations (3.10.1) 45 | diff-lcs (>= 1.2.0, < 2.0) 46 | rspec-support (~> 3.10.0) 47 | rspec-mocks (3.10.2) 48 | diff-lcs (>= 1.2.0, < 2.0) 49 | rspec-support (~> 3.10.0) 50 | rspec-support (3.10.2) 51 | sqlite3 (1.4.2) 52 | thread_safe (0.3.6) 53 | tzinfo (1.2.9) 54 | thread_safe (~> 0.1) 55 | zeitwerk (2.4.2) 56 | 57 | PLATFORMS 58 | x86_64-darwin-19 59 | 60 | DEPENDENCIES 61 | activerecord (~> 6.0.0) 62 | bundler 63 | byebug 64 | database_cleaner 65 | db-query-matchers 66 | jit_preloader! 67 | rake (~> 13.0) 68 | rspec 69 | sqlite3 70 | 71 | BUNDLED WITH 72 | 2.2.12 73 | -------------------------------------------------------------------------------- /Gemfile.6.1: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem "activerecord", "~>6.1" 4 | 5 | # Specify your gem's dependencies in jit_preloader.gemspec 6 | gemspec 7 | -------------------------------------------------------------------------------- /Gemfile.6.1.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | jit_preloader (1.0.3) 5 | activerecord (>= 5.2, < 7) 6 | activesupport 7 | 8 | GEM 9 | remote: https://rubygems.org/ 10 | specs: 11 | activemodel (6.1.4.1) 12 | activesupport (= 6.1.4.1) 13 | activerecord (6.1.4.1) 14 | activemodel (= 6.1.4.1) 15 | activesupport (= 6.1.4.1) 16 | activesupport (6.1.4.1) 17 | concurrent-ruby (~> 1.0, >= 1.0.2) 18 | i18n (>= 1.6, < 2) 19 | minitest (>= 5.1) 20 | tzinfo (~> 2.0) 21 | zeitwerk (~> 2.3) 22 | byebug (11.1.3) 23 | concurrent-ruby (1.1.9) 24 | database_cleaner (2.0.1) 25 | database_cleaner-active_record (~> 2.0.0) 26 | database_cleaner-active_record (2.0.1) 27 | activerecord (>= 5.a) 28 | database_cleaner-core (~> 2.0.0) 29 | database_cleaner-core (2.0.1) 30 | db-query-matchers (0.10.0) 31 | activesupport (>= 4.0, < 7) 32 | rspec (~> 3.0) 33 | diff-lcs (1.4.4) 34 | i18n (1.8.10) 35 | concurrent-ruby (~> 1.0) 36 | minitest (5.14.4) 37 | rake (13.0.6) 38 | rspec (3.10.0) 39 | rspec-core (~> 3.10.0) 40 | rspec-expectations (~> 3.10.0) 41 | rspec-mocks (~> 3.10.0) 42 | rspec-core (3.10.1) 43 | rspec-support (~> 3.10.0) 44 | rspec-expectations (3.10.1) 45 | diff-lcs (>= 1.2.0, < 2.0) 46 | rspec-support (~> 3.10.0) 47 | rspec-mocks (3.10.2) 48 | diff-lcs (>= 1.2.0, < 2.0) 49 | rspec-support (~> 3.10.0) 50 | rspec-support (3.10.2) 51 | sqlite3 (1.4.2) 52 | tzinfo (2.0.4) 53 | concurrent-ruby (~> 1.0) 54 | zeitwerk (2.4.2) 55 | 56 | PLATFORMS 57 | ruby 58 | x86_64-darwin-19 59 | 60 | DEPENDENCIES 61 | activerecord (~> 6.1) 62 | bundler 63 | byebug 64 | database_cleaner 65 | db-query-matchers 66 | jit_preloader! 67 | rake (~> 13.0) 68 | rspec 69 | sqlite3 70 | 71 | BUNDLED WITH 72 | 2.2.12 73 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Kyle d'Oliveira 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JitPreloader 2 | 3 | N+1 queries are a silent killer for performance. Sometimes they can be noticeable; other times they're just a minor tax. We want a way to remove them. 4 | 5 | Imagine you have contacts that have many emails, phone numbers, and addresses. You might have code like this: 6 | 7 | ```ruby 8 | def do_my_thing(contact) 9 | contact.emails.each do |email| 10 | # Do a thing with the email 11 | end 12 | contact.phone_numbers.each do |phone_number| 13 | # Do a thing with the phone number 14 | end 15 | end 16 | 17 | # This will generate two N+1 queries, one for emails and one for phone numbers. 18 | Contact.all.each do |contact| 19 | do_my_thing(contact) 20 | end 21 | ``` 22 | 23 | Rails solves this with `includes` (or better, `preload`/`eager_load`, as they are what `includes` uses in the background). So to get around this problem in Rails you would do something like this: 24 | 25 | ```ruby 26 | Contact.preload(:emails, :phone_numbers).each do |contact| 27 | do_my_thing(contact) 28 | end 29 | ``` 30 | 31 | However this does have some limitations. 32 | 33 | 1) When doing the `preload`, you have to understand what the code does in order to properly load the associations. When this is a brand new method or a simple method this may be simple, but sometimes it can be difficult or time-consuming to figure this out. 34 | 35 | 2) Imagine we change the method to also use the `addresses` association: 36 | 37 | ```ruby 38 | def do_my_thing(contact) 39 | contact.emails.each do |email| 40 | # Do a thing with the email 41 | end 42 | contact.phone_numbers.each do |phone_number| 43 | # Do a thing with the phone number 44 | end 45 | contact.addresses.each do |address| 46 | # Do a thing with the address 47 | end 48 | end 49 | ``` 50 | 51 | All of a sudden we have an N+1 query again. So now you need to go hunt down all of the places were you were preloading and preload the new association. 52 | 53 | 3) Imagine we change the method to do this: 54 | 55 | ```ruby 56 | def do_my_thing(contact) 57 | contact.emails.each do |email| 58 | # Do a thing with the email 59 | end 60 | end 61 | ``` 62 | 63 | We don't have an N+1 query here, but now we are preloading the `phone_numbers` association but not doing anything with it. This is still bad, especially when there are a lot of associations on the object. 64 | 65 | This gem provides a "magic bullet" that can remove most N+1 queries in the application. 66 | 67 | ## Installation 68 | 69 | Add this line to your application's Gemfile: 70 | 71 | ```ruby 72 | gem 'jit_preloader' 73 | ``` 74 | 75 | And then execute: 76 | 77 | $ bundle 78 | 79 | Or install it yourself as: 80 | 81 | $ gem install jit_preloader 82 | 83 | ## Usage 84 | 85 | This gem provides three features: 86 | 87 | ### N+1 query tracking 88 | 89 | This gem will publish an `n_plus_one_query` event via ActiveSupport::Notifications whenever it detects one. This lets you do a variety of useful things. Here are some examples: 90 | 91 | You could implement some basic tracking. This will let you measure the extent of the N+1 query problems in your app: 92 | ```ruby 93 | ActiveSupport::Notifications.subscribe("n_plus_one_query") do |event, data| 94 | statsd.increment "web.#{Rails.env}.n_plus_one_queries.global" 95 | end 96 | ``` 97 | 98 | You could log the N+1 queries. In your development environment, you could throw N+1 queries into the logs along with a stack trace: 99 | ```ruby 100 | ActiveSupport::Notifications.subscribe("n_plus_one_query") do |event, data| 101 | message = "N+1 Query detected: #{data[:association]} on #{data[:source].class}" 102 | backtrace = caller.select{|r| r.starts_with?(Rails.root.to_s) } 103 | Rails.logger.debug("\n\n#{message}\n#{backtrace.join("\n")}\n".red) 104 | end 105 | ``` 106 | 107 | If you use rspec, you could wrap your specs in an `around(:each)` that throws an exception if an N+1 query is detected. You could even provide a tag that allows tests that have known N+1 queries to still pass: 108 | ```ruby 109 | config.around(:each) do |example| 110 | callback = ->(event, data) do 111 | unless example.metadata[:known_n_plus_one_query] 112 | message = "N+1 Query detected: #{data[:source].class} on #{data[:association]}" 113 | raise QueryError.new(message) 114 | end 115 | end 116 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 117 | example.run 118 | end 119 | end 120 | ``` 121 | 122 | ### Jit preloading on a case-by-case basis 123 | 124 | There is now a `jit_preload` and `jit_preload!` method on ActiveRecord::Relation objects. This means instead of using `includes`, `preload` or `eager_load` with the association you want to load, you can simply just use `jit_preload` 125 | 126 | ```ruby 127 | # old 128 | Contact.preload(:addresses, :email_addresses).each do |contact| 129 | contact.addresses.to_a 130 | contact.email_addresses.to_a 131 | end 132 | 133 | # new 134 | Contact.jit_preload.each do |contact| 135 | contact.addresses.to_a 136 | contact.email_addresses.to_a 137 | end 138 | ``` 139 | 140 | ### Loading aggregate methods on associations 141 | 142 | There is now a `has_many_aggregate` method available for ActiveRecord::Base. This will dynamically create a method available on objects that will allow making aggregate queries for a collection. 143 | 144 | ```ruby 145 | # old 146 | Contact.all.each do |contact| 147 | contact.addresses.maximum("LENGTH(street)") 148 | contact.addresses.count 149 | end 150 | # SELECT * FROM contacts 151 | # SELECT MAX(LENGTH(street)) FROM addresses WHERE contact_id = 1 152 | # SELECT COUNT(*) FROM addresses WHERE contact_id = 1 153 | # SELECT MAX(LENGTH(street)) FROM addresses WHERE contact_id = 2 154 | # SELECT COUNT(*) FROM addresses WHERE contact_id = 2 155 | # SELECT MAX(LENGTH(street)) FROM addresses WHERE contact_id = 3 156 | # SELECT COUNT(*) FROM addresses WHERE contact_id = 3 157 | # ... 158 | 159 | #new 160 | class Contact < ActiveRecord::Base 161 | has_many :addresses 162 | has_many_aggregate :addresses, :max_street_length, :maximum, "LENGTH(street)", default: nil 163 | has_many_aggregate :addresses, :count_all, :count, "*" 164 | end 165 | 166 | Contact.jit_preload.each do |contact| 167 | contact.addresses_max_street_length 168 | contact.addresses_count_all 169 | end 170 | # SELECT * FROM contacts 171 | # SELECT contact_id, MAX(LENGTH(street)) FROM addresses WHERE contact_id IN (1, 2, 3, ...) GROUP BY contact_id 172 | # SELECT contact_id, COUNT(*) FROM addresses WHERE contact_id IN (1, 2, 3, ...) GROUP BY contact_id 173 | 174 | ``` 175 | 176 | Furthermore, there is an argument `max_ids_per_query` setting max ids per query. This helps prevent running a single query with too large list of ids which may be less efficient than splitting into multiple queries. 177 | ```ruby 178 | class Contact < ActiveRecord::Base 179 | has_many :addresses 180 | has_many_aggregate :addresses, :count_all, :count, "*", max_ids_per_query: 10 181 | end 182 | 183 | Contact.jit_preload.each do |contact| 184 | contact.addresses_count_all 185 | end 186 | # SELECT contact_id, COUNT(*) FROM addresses WHERE contact_id IN (1, 2, 3, ... ,10) GROUP BY contact_id 187 | # SELECT contact_id, COUNT(*) FROM addresses WHERE contact_id IN (11, 12, 13) GROUP BY contact_id 188 | ``` 189 | 190 | ### Preloading a subset of an association 191 | 192 | There are often times when you want to preload a subset of an association, or change how the SQL statement is generated. For example, if a `Contact` model has 193 | an `addresses` association, you may want to be able to get all of the addresses that belong to a specific country without introducing an N+1 query. 194 | This is a method `preload_scoped_relation` that is available that can handle this for you. 195 | 196 | ```ruby 197 | #old 198 | class Contact < ActiveRecord::Base 199 | has_many :addresses 200 | has_many :usa_addresses, ->{ where(country: Country.find_by_name("USA")) } 201 | end 202 | 203 | Contact.jit_preload.all.each do |contact| 204 | # This will preload the association as expected, but it must be defined as an association in advance 205 | contact.usa_addresses 206 | 207 | # This will preload as the entire addresses association, and filters it in memory 208 | contact.addresses.select{|address| address.country == Country.find_by_name("USA") } 209 | 210 | # This is an N+1 query 211 | contact.addresses.where(country: Country.find_by_name("USA")) 212 | end 213 | 214 | # New 215 | Contact.jit_preload.all.each do |contact| 216 | contact.preload_scoped_relation( 217 | name: "USA Addresses", 218 | base_association: :addresses, 219 | preload_scope: Address.where(country: Country.find_by_name("USA")) 220 | ) 221 | end 222 | # SELECT * FROM contacts 223 | # SELECT * FROM countries WHERE name = "USA" LIMIT 1 224 | # SELECT "addresses".* FROM "addresses" WHERE "addresses"."country_id" = 10 AND "addresses"."contact_id" IN (1, 2, 3, ...) 225 | ``` 226 | 227 | ### Jit preloading globally across your application 228 | 229 | The JitPreloader can be globally enabled, in which case most N+1 queries in your app should just disappear. It is off by default. 230 | The `max_ids_per_query` argument on loading aggregate methods can also apply on a global level. 231 | 232 | ```ruby 233 | # Can be true or false 234 | JitPreloader.globally_enabled = true 235 | 236 | # Can also be given anything that responds to `call`. 237 | # You could build a kill switch with Redis (or whatever you'd like) 238 | # so that you can turn it on or off dynamically. 239 | JitPreloader.globally_enabled = ->{ $redis.get('always_jit_preload') == 'on' } 240 | 241 | # Setting global max ids constraint on all aggregation methods. 242 | JitPreloader.max_ids_per_query = 10 243 | 244 | class Contact < ActiveRecord::Base 245 | has_many :emails 246 | has_many_aggregate :emails, :count_all, :count, "*" 247 | end 248 | 249 | # When enabled globally, this would not generate an N+1 query. 250 | Contact.all.each do |contact| 251 | contact.emails.each do |email| 252 | # do something 253 | end 254 | # When max_ides_per_query is set globally, the aggregate method will split query base on the limit. 255 | contact.emails_count_all 256 | end 257 | 258 | 259 | ``` 260 | 261 | ## What it doesn't solve 262 | 263 | This is mostly a magic bullet, but it doesn't solve all database-related problems. If you reload an association, or call a query or aggregate function on the association, it will not remove those extra queries. These problems cannot be solved by using Rails' `preload` so it cannot be solved with the Jit Preloader. 264 | 265 | ```ruby 266 | Contact.all.each do |contact| 267 | contact.emails.reload # Reloading the association 268 | contact.addresses.where(billing: true).to_a # Querying the association (Use: preload_scoped_relation to avoid these) 269 | end 270 | ``` 271 | 272 | ## Consequences 273 | 274 | 1) This gem introduces more Magic. This is fine, but you should really understand what is going on under the hood. You should understand what makes an N+1 query happen and what this gem is doing to help address it. 275 | 276 | 2) We may do more work than you require. If you have turned the preloader on globally but you only want to access a single record's association, it will load the association for the entire collection you were looking at. 277 | 278 | 3) Each result set will have a JitPreloader setup on it, and the preloader will have a reference to all of the other objects in a result set. This means that so long as one object of that result set exists in memory, the others will not be cleaned up by the garbage collector. This shouldn't have much impact, but it's good to be aware of it. 279 | 280 | ## Contributing 281 | 282 | 1. Fork it ( https://github.com/clio/jit_preloader/fork ) 283 | 2. Create your feature branch (`git checkout -b my-new-feature`) 284 | 3. Commit your changes (`git commit -am 'Add some feature'`) 285 | 4. Push to the branch (`git push origin my-new-feature`) 286 | 5. Create a new Pull Request 287 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | -------------------------------------------------------------------------------- /jit_preloader.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'jit_preloader/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "jit_preloader" 8 | spec.version = JitPreloader::VERSION 9 | spec.authors = ["Kyle d'Oliveira"] 10 | spec.email = ["kyle.doliveira@clio.com"] 11 | spec.summary = %q{Tool to understand N+1 queries and to remove them} 12 | spec.description = %q{The JitPreloader has the ability to send notifications when N+1 queries occur to help guage how problematic they are for your code base and a way to remove all of the commons explicitly or automatically} 13 | spec.homepage = "https://github.com/clio/jit_preloader" 14 | spec.metadata["homepage_uri"] = spec.homepage 15 | spec.metadata["source_code_uri"] = spec.homepage 16 | 17 | spec.license = "MIT" 18 | 19 | spec.files = `git ls-files -z`.split("\x0") 20 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 21 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 22 | spec.require_paths = ["lib"] 23 | 24 | spec.add_dependency "activerecord", "< 8" 25 | spec.add_dependency "activesupport" 26 | 27 | spec.add_development_dependency "bundler" 28 | spec.add_development_dependency "rake", "~> 13.0" 29 | spec.add_development_dependency "rspec" 30 | spec.add_development_dependency "database_cleaner" 31 | spec.add_development_dependency "sqlite3" 32 | spec.add_development_dependency "byebug" 33 | spec.add_development_dependency "db-query-matchers" 34 | end 35 | -------------------------------------------------------------------------------- /lib/jit_preloader.rb: -------------------------------------------------------------------------------- 1 | require 'active_support/concern' 2 | require 'active_support/core_ext/module/delegation' 3 | require 'active_support/notifications' 4 | require 'active_record' 5 | 6 | require "jit_preloader/version" 7 | require 'jit_preloader/active_record/base' 8 | require 'jit_preloader/active_record/relation' 9 | require 'jit_preloader/active_record/associations/collection_association' 10 | require 'jit_preloader/active_record/associations/singular_association' 11 | if Gem::Version.new(ActiveRecord::VERSION::STRING) >= Gem::Version.new("7.0.0") 12 | require 'jit_preloader/active_record/associations/preloader/ar7_association' 13 | require 'jit_preloader/active_record/associations/preloader/ar7_branch' 14 | elsif Gem::Version.new(ActiveRecord::VERSION::STRING) >= Gem::Version.new("6.1.0") 15 | require 'jit_preloader/active_record/associations/preloader/ar6_association' 16 | else 17 | require 'jit_preloader/active_record/associations/preloader/collection_association' 18 | require 'jit_preloader/active_record/associations/preloader/singular_association' 19 | end 20 | require 'jit_preloader/preloader' 21 | 22 | module JitPreloader 23 | def self.globally_enabled=(value) 24 | @enabled = value 25 | end 26 | 27 | def self.max_ids_per_query=(max_ids) 28 | if max_ids && max_ids >= 1 29 | @max_ids_per_query = max_ids 30 | end 31 | end 32 | 33 | def self.max_ids_per_query 34 | @max_ids_per_query 35 | end 36 | 37 | def self.globally_enabled? 38 | if @enabled && @enabled.respond_to?(:call) 39 | @enabled.call 40 | else 41 | @enabled 42 | end 43 | end 44 | 45 | end 46 | -------------------------------------------------------------------------------- /lib/jit_preloader/active_record/associations/collection_association.rb: -------------------------------------------------------------------------------- 1 | module JitPreloader 2 | module ActiveRecordAssociationsCollectionAssociation 3 | 4 | def load_target 5 | was_loaded = loaded? 6 | 7 | if !loaded? && owner.persisted? && owner.jit_preloader && (reflection.scope.nil? || reflection.scope.arity == 0) 8 | owner.jit_preloader.jit_preload(reflection.name) 9 | end 10 | 11 | jit_loaded = loaded? 12 | 13 | super.tap do |records| 14 | # We should not act on non-persisted objects, or ones that are already loaded. 15 | if owner.persisted? && !was_loaded 16 | # If we went through a JIT preload, then we will have attached another JitPreloader elsewhere. 17 | JitPreloader::Preloader.attach(records) if records.any? && !jit_loaded && JitPreloader.globally_enabled? 18 | 19 | # If the records were not pre_loaded 20 | records.each{ |record| record.jit_n_plus_one_tracking = true } 21 | 22 | if !jit_loaded && owner.jit_n_plus_one_tracking 23 | ActiveSupport::Notifications.publish("n_plus_one_query", 24 | source: owner, association: reflection.name) 25 | end 26 | end 27 | end 28 | end 29 | end 30 | end 31 | 32 | ActiveRecord::Associations::CollectionAssociation.prepend(JitPreloader::ActiveRecordAssociationsCollectionAssociation) 33 | -------------------------------------------------------------------------------- /lib/jit_preloader/active_record/associations/preloader/ar6_association.rb: -------------------------------------------------------------------------------- 1 | module JitPreloader 2 | module PreloaderAssociation 3 | 4 | # A monkey patch to ActiveRecord. The old method looked like the snippet 5 | # below. Our changes here are that we remove records that are already 6 | # part of the target, then attach all of the records to a new jit preloader. 7 | # 8 | # def run 9 | # records = records_by_owner 10 | 11 | # owners.each do |owner| 12 | # associate_records_to_owner(owner, records[owner] || []) 13 | # end if @associate 14 | 15 | # self 16 | # end 17 | 18 | def run 19 | return unless (reflection.scope.nil? || reflection.scope.arity == 0) && klass.ancestors.include?(ActiveRecord::Base) 20 | 21 | super.tap do 22 | if preloaded_records.any? && preloaded_records.none?(&:jit_preloader) 23 | JitPreloader::Preloader.attach(preloaded_records) if owners.any?(&:jit_preloader) || JitPreloader.globally_enabled? 24 | end 25 | end 26 | end 27 | 28 | # Original method: 29 | # def associate_records_to_owner(owner, records) 30 | # association = owner.association(reflection.name) 31 | # if reflection.collection? 32 | # association.target = records 33 | # else 34 | # association.target = records.first 35 | # end 36 | # end 37 | def associate_records_to_owner(owner, records) 38 | association = owner.association(reflection.name) 39 | if reflection.collection? 40 | new_records = association.target.any? ? records - association.target : records 41 | association.target.concat(new_records) 42 | association.loaded! 43 | else 44 | association.target = records.first 45 | end 46 | end 47 | 48 | def build_scope 49 | super.tap do |scope| 50 | scope.jit_preload! if owners.any?(&:jit_preloader) || JitPreloader.globally_enabled? 51 | end 52 | end 53 | end 54 | end 55 | 56 | ActiveRecord::Associations::Preloader::Association.prepend(JitPreloader::PreloaderAssociation) 57 | ActiveRecord::Associations::Preloader::ThroughAssociation.prepend(JitPreloader::PreloaderAssociation) 58 | -------------------------------------------------------------------------------- /lib/jit_preloader/active_record/associations/preloader/ar7_association.rb: -------------------------------------------------------------------------------- 1 | module JitPreloader 2 | module PreloaderAssociation 3 | 4 | # A monkey patch to ActiveRecord. The old method looked like the snippet 5 | # below. Our changes here are that we remove records that are already 6 | # part of the target, then attach all of the records to a new jit preloader. 7 | # 8 | # def run 9 | # records = records_by_owner 10 | 11 | # owners.each do |owner| 12 | # associate_records_to_owner(owner, records[owner] || []) 13 | # end if @associate 14 | 15 | # self 16 | # end 17 | 18 | def run 19 | return unless (reflection.scope.nil? || reflection.scope.arity == 0) && klass.ancestors.include?(ActiveRecord::Base) 20 | 21 | super.tap do 22 | if preloaded_records.any? && preloaded_records.none?(&:jit_preloader) 23 | JitPreloader::Preloader.attach(preloaded_records) if owners.any?(&:jit_preloader) || JitPreloader.globally_enabled? 24 | end 25 | end 26 | end 27 | 28 | # Original method: 29 | # def associate_records_to_owner(owner, records) 30 | # return if loaded?(owner) 31 | # 32 | # association = owner.association(reflection.name) 33 | # 34 | # if reflection.collection? 35 | # association.target = records 36 | # else 37 | # association.target = records.first 38 | # end 39 | # end 40 | def associate_records_to_owner(owner, records) 41 | return if loaded?(owner) 42 | 43 | association = owner.association(reflection.name) 44 | 45 | if reflection.collection? 46 | new_records = association.target.any? ? records - association.target : records 47 | association.target.concat(new_records) 48 | association.loaded! 49 | else 50 | association.target = records.first 51 | end 52 | end 53 | 54 | def build_scope 55 | super.tap do |scope| 56 | scope.jit_preload! if owners.any?(&:jit_preloader) || JitPreloader.globally_enabled? 57 | end 58 | end 59 | end 60 | end 61 | 62 | ActiveRecord::Associations::Preloader::Association.prepend(JitPreloader::PreloaderAssociation) 63 | ActiveRecord::Associations::Preloader::ThroughAssociation.prepend(JitPreloader::PreloaderAssociation) 64 | -------------------------------------------------------------------------------- /lib/jit_preloader/active_record/associations/preloader/ar7_branch.rb: -------------------------------------------------------------------------------- 1 | module JitPreloader 2 | module PreloaderBranch 3 | """ 4 | ActiveRecord version >= 7.x.x introduced an improvement for preloading associations in batches: 5 | https://github.com/rails/rails/blob/main/activerecord/lib/active_record/associations/preloader.rb#L121 6 | 7 | Our existing monkey-patches will ignore associations whose classes are not descendants of 8 | ActiveRecord::Base (example: https://github.com/clio/jit_preloader/blob/master/lib/jit_preloader/active_record/associations/preloader/ar6_association.rb#L19). 9 | But this change breaks that behaviour because now Batch is calling `klass.base_class` (a method defined by ActiveRecord::Base) 10 | before we have a chance to filter out the non-AR classes. 11 | This patch for AR 7.x makes the Branch class ignore any association loaders that aren't for ActiveRecord::Base subclasses. 12 | """ 13 | 14 | def loaders 15 | @loaders = super.find_all do |loader| 16 | loader.klass < ::ActiveRecord::Base 17 | end 18 | end 19 | end 20 | end 21 | 22 | ActiveRecord::Associations::Preloader::Branch.prepend(JitPreloader::PreloaderBranch) 23 | -------------------------------------------------------------------------------- /lib/jit_preloader/active_record/associations/preloader/collection_association.rb: -------------------------------------------------------------------------------- 1 | class ActiveRecord::Associations::Preloader::CollectionAssociation 2 | private 3 | # A monkey patch to ActiveRecord. The old method looked like the snippet 4 | # below. Our changes here are that we remove records that are already 5 | # part of the target, then attach all of the records to a new jit preloader. 6 | # 7 | # def preload(preloader) 8 | # associated_records_by_owner(preloader).each do |owner, records| 9 | # association = owner.association(reflection.name) 10 | # association.loaded! 11 | # association.target.concat(records) 12 | # records.each { |record| association.set_inverse_instance(record) } 13 | # end 14 | # end 15 | 16 | def preload(preloader) 17 | return unless (reflection.scope.nil? || reflection.scope.arity == 0) && klass.ancestors.include?(ActiveRecord::Base) 18 | all_records = [] 19 | associated_records_by_owner(preloader).each do |owner, records| 20 | association = owner.association(reflection.name) 21 | association.loaded! 22 | # It is possible that some of the records are loaded already. 23 | # We don't want to duplicate them, but we also want to preserve 24 | # the original copy so that we don't blow away in-memory changes. 25 | new_records = association.target.any? ? records - association.target : records 26 | 27 | association.target.concat(new_records) 28 | new_records.each { |record| association.set_inverse_instance(record) } 29 | 30 | all_records.concat(records) if owner.jit_preloader || JitPreloader.globally_enabled? 31 | end 32 | JitPreloader::Preloader.attach(all_records) if all_records.any? 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/jit_preloader/active_record/associations/preloader/singular_association.rb: -------------------------------------------------------------------------------- 1 | class ActiveRecord::Associations::Preloader::SingularAssociation 2 | private 3 | # A monkey patch to ActiveRecord. The old method looked like the snippet 4 | # below. Our changes here are that we don't assign the record if the 5 | # target has already been set, and we attach all of the records to a new 6 | # jit preloader. 7 | # 8 | # def preload(preloader) 9 | # associated_records_by_owner(preloader).each do |owner, associated_records| 10 | # record = associated_records.first 11 | # association = owner.association(reflection.name) 12 | # association.target = record 13 | # end 14 | # end 15 | 16 | def preload(preloader) 17 | return unless (reflection.scope.nil? || reflection.scope.arity == 0) && klass.ancestors.include?(ActiveRecord::Base) 18 | all_records = [] 19 | 20 | associated_records_by_owner(preloader).each do |owner, associated_records| 21 | record = associated_records.first 22 | 23 | association = owner.association(reflection.name) 24 | association.target ||= record 25 | all_records.push(record) if record && (owner.jit_preloader || JitPreloader.globally_enabled?) 26 | end 27 | JitPreloader::Preloader.attach(all_records) if all_records.any? 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/jit_preloader/active_record/associations/singular_association.rb: -------------------------------------------------------------------------------- 1 | module JitPreloader 2 | module ActiveRecordAssociationsSingularAssociation 3 | 4 | def load_target 5 | was_loaded = loaded? 6 | 7 | if !loaded? && owner.persisted? && owner.jit_preloader && (reflection.scope.nil? || reflection.scope.arity == 0) 8 | owner.jit_preloader.jit_preload(reflection.name) 9 | end 10 | 11 | jit_loaded = loaded? 12 | 13 | super.tap do |record| 14 | if owner.persisted? && !was_loaded 15 | # If the owner doesn't track N+1 queries, then we don't need to worry about 16 | # tracking it on the record. This is because you can do something like: 17 | # model.foo.bar (where foo and bar are singular associations) and that isn't 18 | # always an N+1 query. 19 | record.jit_n_plus_one_tracking ||= owner.jit_n_plus_one_tracking if record 20 | 21 | if !jit_loaded && owner.jit_n_plus_one_tracking && !is_polymorphic_association_without_type 22 | ActiveSupport::Notifications.publish("n_plus_one_query", 23 | source: owner, association: reflection.name) 24 | end 25 | end 26 | end 27 | end 28 | 29 | private def is_polymorphic_association_without_type 30 | self.is_a?(ActiveRecord::Associations::BelongsToPolymorphicAssociation) && self.klass.nil? 31 | end 32 | end 33 | end 34 | 35 | ActiveRecord::Associations::SingularAssociation.prepend(JitPreloader::ActiveRecordAssociationsSingularAssociation) 36 | -------------------------------------------------------------------------------- /lib/jit_preloader/active_record/base.rb: -------------------------------------------------------------------------------- 1 | module JitPreloadExtension 2 | attr_accessor :jit_preloader 3 | attr_accessor :jit_n_plus_one_tracking 4 | attr_accessor :jit_preload_aggregates 5 | attr_accessor :jit_preload_scoped_relations 6 | 7 | def reload(*args) 8 | clear_jit_preloader! 9 | super 10 | end 11 | 12 | def clear_jit_preloader! 13 | self.jit_preload_aggregates = {} 14 | self.jit_preload_scoped_relations = {} 15 | if jit_preloader 16 | jit_preloader.records.delete(self) 17 | self.jit_preloader = nil 18 | end 19 | end 20 | 21 | if Gem::Version.new(ActiveRecord::VERSION::STRING) >= Gem::Version.new("7.0.0") 22 | def preload_scoped_relation(name:, base_association:, preload_scope: nil) 23 | return jit_preload_scoped_relations[name] if jit_preload_scoped_relations&.key?(name) 24 | 25 | base_association = base_association.to_sym 26 | records = jit_preloader&.records || [self] 27 | previous_association_values = {} 28 | 29 | records.each do |record| 30 | association = record.association(base_association) 31 | if association.loaded? 32 | previous_association_values[record] = association.target 33 | association.reset 34 | end 35 | end 36 | 37 | preloader_association = ActiveRecord::Associations::Preloader.new( 38 | records: records, 39 | associations: base_association, 40 | scope: preload_scope 41 | ).call.first 42 | 43 | records.each do |record| 44 | record.jit_preload_scoped_relations ||= {} 45 | association = record.association(base_association) 46 | record.jit_preload_scoped_relations[name] = preloader_association.records_by_owner[record] || [] 47 | association.reset 48 | if previous_association_values.key?(record) 49 | association.target = previous_association_values[record] 50 | end 51 | end 52 | 53 | jit_preload_scoped_relations[name] 54 | end 55 | else 56 | def preload_scoped_relation(name:, base_association:, preload_scope: nil) 57 | return jit_preload_scoped_relations[name] if jit_preload_scoped_relations&.key?(name) 58 | 59 | base_association = base_association.to_sym 60 | records = jit_preloader&.records || [self] 61 | previous_association_values = {} 62 | 63 | records.each do |record| 64 | association = record.association(base_association) 65 | if association.loaded? 66 | previous_association_values[record] = association.target 67 | association.reset 68 | end 69 | end 70 | 71 | ActiveRecord::Associations::Preloader.new.preload( 72 | records, 73 | base_association, 74 | preload_scope 75 | ) 76 | 77 | records.each do |record| 78 | record.jit_preload_scoped_relations ||= {} 79 | association = record.association(base_association) 80 | record.jit_preload_scoped_relations[name] = association.target 81 | association.reset 82 | if previous_association_values.key?(record) 83 | association.target = previous_association_values[record] 84 | end 85 | end 86 | 87 | jit_preload_scoped_relations[name] 88 | end 89 | end 90 | 91 | def self.prepended(base) 92 | class << base 93 | delegate :jit_preload, to: :all 94 | 95 | def has_many_aggregate(assoc, name, aggregate, field, table_alias_name: nil, default: 0, max_ids_per_query: nil) 96 | method_name = "#{assoc}_#{name}" 97 | 98 | define_method(method_name) do |conditions={}| 99 | self.jit_preload_aggregates ||= {} 100 | 101 | key = "#{method_name}|#{conditions.sort.hash}" 102 | return jit_preload_aggregates[key] if jit_preload_aggregates.key?(key) 103 | if jit_preloader 104 | reflection = association(assoc).reflection 105 | primary_ids = jit_preloader.records.collect{|r| r[reflection.active_record_primary_key] } 106 | max_ids_per_query = max_ids_per_query || JitPreloader.max_ids_per_query 107 | if max_ids_per_query 108 | slices = primary_ids.each_slice(max_ids_per_query) 109 | else 110 | slices = [primary_ids] 111 | end 112 | 113 | klass = reflection.klass 114 | 115 | aggregate_association = reflection 116 | while aggregate_association.through_reflection 117 | aggregate_association = aggregate_association.through_reflection 118 | end 119 | 120 | association_scope = klass.all.merge(association(assoc).scope).unscope(where: aggregate_association.foreign_key) 121 | association_scope = association_scope.instance_exec(&reflection.scope).reorder(nil) if reflection.scope 122 | 123 | # If the query uses an alias for the association, use that instead of the table name 124 | table_reference = table_alias_name 125 | table_reference ||= association_scope.references_values.first || aggregate_association.table_name 126 | 127 | # If the association is a STI child model, specify its type in the condition so that it 128 | # doesn't include results from other child models 129 | parent_is_base_class = aggregate_association.klass.superclass.abstract_class? || aggregate_association.klass.superclass == ActiveRecord::Base 130 | has_type_column = aggregate_association.klass.column_names.include?(aggregate_association.klass.inheritance_column) 131 | is_child_sti_model = !parent_is_base_class && has_type_column 132 | if is_child_sti_model 133 | conditions[table_reference] = { aggregate_association.klass.inheritance_column => aggregate_association.klass.sti_name } 134 | end 135 | 136 | if reflection.type.present? 137 | conditions[reflection.type] = self.class.name 138 | end 139 | group_by = "#{table_reference}.#{aggregate_association.foreign_key}" 140 | 141 | preloaded_data = {} 142 | slices.each do |slice| 143 | data = Hash[association_scope 144 | .where(conditions.deep_merge(table_reference => { aggregate_association.foreign_key => slice })) 145 | .group(group_by) 146 | .send(aggregate, field) 147 | ] 148 | preloaded_data.merge!(data) 149 | end 150 | 151 | jit_preloader.records.each do |record| 152 | record.jit_preload_aggregates ||= {} 153 | record.jit_preload_aggregates[key] = preloaded_data[record.id] || default 154 | end 155 | else 156 | self.jit_preload_aggregates[key] = send(assoc).where(conditions).send(aggregate, field) || default 157 | end 158 | jit_preload_aggregates[key] 159 | end 160 | end 161 | end 162 | end 163 | end 164 | 165 | ActiveRecord::Base.send(:prepend, JitPreloadExtension) 166 | -------------------------------------------------------------------------------- /lib/jit_preloader/active_record/relation.rb: -------------------------------------------------------------------------------- 1 | module JitPreloader 2 | module ActiveRecordRelation 3 | 4 | def jit_preload(*args) 5 | spawn.jit_preload!(*args) 6 | end 7 | 8 | def jit_preload!(*args) 9 | @jit_preload = true 10 | self 11 | end 12 | 13 | def jit_preload? 14 | @jit_preload 15 | end 16 | 17 | def calculate(*args) 18 | if respond_to?(:proxy_association) && proxy_association.owner && proxy_association.owner.jit_n_plus_one_tracking 19 | ActiveSupport::Notifications.publish("n_plus_one_query", 20 | source: proxy_association.owner, 21 | association: "#{proxy_association.reflection.name}.#{args.first}") 22 | end 23 | 24 | super(*args) 25 | end 26 | 27 | def exec_queries 28 | super.tap do |records| 29 | if limit_value != 1 30 | records.each{ |record| record.jit_n_plus_one_tracking = true } 31 | if jit_preload? || JitPreloader.globally_enabled? 32 | JitPreloader::Preloader.attach(records) 33 | end 34 | end 35 | end 36 | end 37 | 38 | end 39 | end 40 | 41 | ActiveRecord::Relation.prepend(JitPreloader::ActiveRecordRelation) 42 | -------------------------------------------------------------------------------- /lib/jit_preloader/preloader.rb: -------------------------------------------------------------------------------- 1 | module JitPreloader 2 | class Preloader < ActiveRecord::Associations::Preloader 3 | 4 | attr_accessor :records 5 | 6 | if Gem::Version.new(ActiveRecord::VERSION::STRING) >= Gem::Version.new("7.0.0") 7 | def self.attach(records) 8 | new(records: records.dup, associations: nil).tap do |loader| 9 | records.each do |record| 10 | record.jit_preloader = loader 11 | end 12 | end 13 | end 14 | 15 | def jit_preload(associations) 16 | # It is possible that the records array has multiple different classes (think single table inheritance). 17 | # Thus, it is possible that some of the records don't have an association. 18 | records_with_association = records.reject{|r| r.class.reflect_on_association(associations).nil? } 19 | 20 | # Some of the records may already have the association loaded and we should not load them again 21 | records_requiring_loading = records_with_association.select{|r| !r.association(associations).loaded? } 22 | 23 | self.class.new(records: records_requiring_loading, associations: associations).call 24 | end 25 | else 26 | def self.attach(records) 27 | new.tap do |loader| 28 | loader.records = records.dup 29 | records.each do |record| 30 | record.jit_preloader = loader 31 | end 32 | end 33 | end 34 | 35 | def jit_preload(associations) 36 | # It is possible that the records array has multiple different classes (think single table inheritance). 37 | # Thus, it is possible that some of the records don't have an association. 38 | records_with_association = records.reject{ |record| record.class.reflect_on_association(associations).nil? } 39 | 40 | # Some of the records may already have the association loaded and we should not load them again 41 | records_requiring_loading = records_with_association.select{ |record| !record.association(associations).loaded? } 42 | preload records_with_association, associations 43 | end 44 | end 45 | 46 | 47 | # We do not want the jit_preloader to be dumpable 48 | # If you dump a ActiveRecord::Base object that has a jit_preloader instance variable 49 | # you will also end up dumping all of the records the preloader has reference to. 50 | # Imagine getting N objects from a query and dumping each one of those into a cache 51 | # each object would dump N+1 objects which means you'll end up storing O(N^2) memory. Thats no good. 52 | # So instead, we will just nullify the jit_preloader on load 53 | def _dump(level) 54 | "" 55 | end 56 | 57 | def self._load(args) 58 | nil 59 | end 60 | 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/jit_preloader/version.rb: -------------------------------------------------------------------------------- 1 | module JitPreloader 2 | VERSION = "3.1.0" 3 | end 4 | -------------------------------------------------------------------------------- /spec/lib/jit_preloader/active_record/base_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require "db-query-matchers" 3 | 4 | RSpec.describe "ActiveRecord::Base Extensions" do 5 | 6 | let(:canada) { Country.create(name: "Canada") } 7 | let(:usa) { Country.create(name: "U.S.A") } 8 | 9 | describe "#preload_scoped_relation" do 10 | def call(contact) 11 | contact.preload_scoped_relation( 12 | name: "American Addresses", 13 | base_association: :addresses, 14 | preload_scope: Address.where(country: usa) 15 | ) 16 | end 17 | 18 | before do 19 | Contact.create(name: "Bar", addresses: [ 20 | Address.new(street: "123 Fake st", country: canada), 21 | Address.new(street: "21 Jump st", country: usa), 22 | Address.new(street: "90210 Beverly Hills", country: usa) 23 | ]) 24 | 25 | Contact.create(name: "Foo", addresses: [ 26 | Address.new(street: "1 First st", country: canada), 27 | Address.new(street: "10 Tenth Ave", country: usa) 28 | ]) 29 | end 30 | 31 | context "when operating on a single object" do 32 | it "will load the objects for that object" do 33 | contact = Contact.first 34 | expect(call(contact)).to match_array contact.addresses.where(country: usa).to_a 35 | end 36 | end 37 | 38 | it "memoizes the result" do 39 | contacts = Contact.jit_preload.limit(2).to_a 40 | 41 | expect do 42 | expect(call(contacts.first)) 43 | expect(call(contacts.first)) 44 | end.to make_database_queries(count: 1) 45 | end 46 | 47 | context "when reloading the object" do 48 | it "clears the memoization" do 49 | contacts = Contact.jit_preload.limit(2).to_a 50 | 51 | expect do 52 | expect(call(contacts.first)) 53 | end.to make_database_queries(count: 1) 54 | contacts.first.reload 55 | expect do 56 | expect(call(contacts.first)) 57 | end.to make_database_queries(count: 1) 58 | end 59 | end 60 | 61 | it "will issue one query for the group of objects" do 62 | contacts = Contact.jit_preload.limit(2).to_a 63 | 64 | usa_addresses = contacts.first.addresses.where(country: usa).to_a 65 | expect do 66 | expect(call(contacts.first)).to match_array usa_addresses 67 | end.to make_database_queries(count: 1) 68 | 69 | usa_addresses = contacts.last.addresses.where(country: usa).to_a 70 | expect do 71 | expect(call(contacts.last)).to match_array usa_addresses 72 | end.to_not make_database_queries 73 | end 74 | 75 | it "doesn't load the value into the association" do 76 | contacts = Contact.jit_preload.limit(2).to_a 77 | expect(contacts.first.association(:addresses)).to_not be_loaded 78 | expect(contacts.last.association(:addresses)).to_not be_loaded 79 | 80 | call(contacts.first) 81 | 82 | expect(contacts.first.association(:addresses)).to_not be_loaded 83 | expect(contacts.last.association(:addresses)).to_not be_loaded 84 | end 85 | 86 | context "when the association is already loaded" do 87 | it "doesn't change the value of the association" do 88 | contacts = Contact.jit_preload.limit(2).to_a 89 | contacts.each{|contact| contact.addresses.to_a } 90 | contacts.each{|contact| call(contact) } 91 | 92 | expect(contacts.first.association(:addresses)).to be_loaded 93 | expect(contacts.last.association(:addresses)).to be_loaded 94 | end 95 | end 96 | 97 | context "when no records exist for the association" do 98 | let!(:record) { Parent.create } 99 | 100 | it "returns an empty array" do 101 | value = record.preload_scoped_relation( 102 | name: "Parent Children", 103 | base_association: :parents_child 104 | ) 105 | 106 | expect(value).to eq([]) 107 | end 108 | end 109 | 110 | context "when preload_scoped_relation with string base_association name" do 111 | it "preload properly" do 112 | contacts = Contact.jit_preload.limit(2).to_a 113 | 114 | call_with_string = lambda { |contact| contact.preload_scoped_relation( 115 | name: "American Addresses", 116 | base_association: "addresses", 117 | preload_scope: Address.where(country: usa) 118 | ) } 119 | 120 | usa_addresses = contacts.first.addresses.where(country: usa).to_a 121 | expect do 122 | expect(call_with_string.call(contacts.first)).to match_array usa_addresses 123 | end.to make_database_queries(count: 1) 124 | 125 | usa_addresses = contacts.last.addresses.where(country: usa).to_a 126 | expect do 127 | expect(call_with_string.call(contacts.last)).to match_array usa_addresses 128 | end.to_not make_database_queries 129 | end 130 | end 131 | end 132 | end 133 | -------------------------------------------------------------------------------- /spec/lib/jit_preloader/preloader_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | require "db-query-matchers" 3 | 4 | RSpec.describe JitPreloader::Preloader do 5 | let!(:contact1) do 6 | addresses = [ 7 | Address.new(street: "123 Fake st", country: canada), 8 | Address.new(street: "21 Jump st", country: usa), 9 | Address.new(street: "90210 Beverly Hills", country: usa) 10 | ] 11 | phones = [ 12 | PhoneNumber.new(phone: "4445556666"), 13 | PhoneNumber.new(phone: "2223333444") 14 | ] 15 | Contact.create(name: "Only Addresses", addresses: addresses, phone_numbers: phones) 16 | end 17 | 18 | let!(:contact2) do 19 | Contact.create(name: "Only Emails", email_address: EmailAddress.new(address: "woot@woot.com")) 20 | end 21 | 22 | let!(:contact3) do 23 | addresses = [ 24 | Address.new(street: "1 First st", country: canada), 25 | Address.new(street: "10 Tenth Ave", country: usa) 26 | ] 27 | Contact.create( 28 | name: "Both!", 29 | addresses: addresses, 30 | email_address: EmailAddress.new(address: "woot@woot.com"), 31 | phone_numbers: [PhoneNumber.new(phone: "1234567890")] 32 | ) 33 | end 34 | 35 | let!(:contact_owner) do 36 | contact3.contact_owner_id = contact1.id 37 | contact3.contact_owner_type = "Address" 38 | contact3.save! 39 | ContactOwner.create( 40 | contacts: [contact1, contact2], 41 | ) 42 | end 43 | 44 | let(:canada) { Country.create(name: "Canada") } 45 | let(:usa) { Country.create(name: "U.S.A") } 46 | 47 | let(:source_map) { Hash.new{|h,k| h[k]= Array.new } } 48 | let(:callback) do 49 | ->(event, data){ source_map[data[:source]] << data[:association] } 50 | end 51 | 52 | context "for single table inheritance" do 53 | context "when preloading an aggregate for a child model" do 54 | let!(:contact_book) { ContactBook.create(name: "The Yellow Pages") } 55 | let!(:company1) { Company.create(name: "Company1", contact_book: contact_book) } 56 | let!(:company2) { Company.create(name: "Company2", contact_book: contact_book) } 57 | 58 | it "can handle queries" do 59 | contact_books = ContactBook.jit_preload.to_a 60 | expect(contact_books.first.companies_count).to eq 2 61 | end 62 | end 63 | 64 | context "when preloading an aggregate of a child model through its base model" do 65 | let!(:contact_book) { ContactBook.create(name: "The Yellow Pages") } 66 | let!(:contact) { Contact.create(name: "Contact", contact_book: contact_book) } 67 | let!(:company1) { Company.create(name: "Company1", contact_book: contact_book) } 68 | let!(:company2) { Company.create(name: "Company2", contact_book: contact_book) } 69 | let!(:contact_employee1) { Employee.create(name: "Contact Employee1", contact: contact) } 70 | let!(:contact_employee2) { Employee.create(name: "Contact Employee2", contact: contact) } 71 | let!(:company_employee1) { Employee.create(name: "Company Employee1", contact: company1) } 72 | let!(:company_employee2) { Employee.create(name: "Company Employee2", contact: company2) } 73 | 74 | it "can handle queries" do 75 | contact_books = ContactBook.jit_preload.to_a 76 | expect(contact_books.first.employees_count).to eq 4 77 | end 78 | end 79 | 80 | context "when preloading an aggregate of a nested child model through another child model" do 81 | let!(:contact_book) { ContactBook.create(name: "The Yellow Pages") } 82 | let!(:contact) { Contact.create(name: "Contact", contact_book: contact_book) } 83 | let!(:company1) { Company.create(name: "Company1", contact_book: contact_book) } 84 | let!(:company2) { Company.create(name: "Company2", contact_book: contact_book) } 85 | let!(:contact_employee1) { Employee.create(name: "Contact Employee1", contact: contact) } 86 | let!(:contact_employee2) { Employee.create(name: "Contact Employee2", contact: contact) } 87 | let!(:company_employee1) { Employee.create(name: "Company Employee1", contact: company1) } 88 | let!(:company_employee2) { Employee.create(name: "Company Employee2", contact: company2) } 89 | 90 | it "can handle queries" do 91 | contact_books = ContactBook.jit_preload.to_a 92 | expect(contact_books.first.company_employees_count).to eq 2 93 | end 94 | end 95 | 96 | context "when preloading an aggregate of a nested child model through a many-to-many relationship with another child model" do 97 | let!(:contact_book) { ContactBook.create(name: "The Yellow Pages") } 98 | let!(:child1) { Child.create(name: "Child1") } 99 | let!(:child2) { Child.create(name: "Child2") } 100 | let!(:child3) { Child.create(name: "Child3") } 101 | let!(:parent1) { Parent.create(name: "Parent1", contact_book: contact_book, children: [child1, child2]) } 102 | let!(:parent2) { Parent.create(name: "Parent2", contact_book: contact_book, children: [child2, child3]) } 103 | 104 | it "can handle queries" do 105 | contact_books = ContactBook.jit_preload.to_a 106 | expect(contact_books.first.children_count).to eq 4 107 | expect(contact_books.first.children).to include(child1, child2, child3) 108 | end 109 | end 110 | 111 | context "when preloading an aggregate for a child model scoped by another join table" do 112 | let!(:contact_book) { ContactBook.create(name: "The Yellow Pages") } 113 | let!(:contact1) { Company.create(name: "Without Email", contact_book: contact_book) } 114 | let!(:contact2) { Company.create(name: "With Blank Email", email_address: EmailAddress.new(address: ""), contact_book: contact_book) } 115 | let!(:contact3) { Company.create(name: "With Email", email_address: EmailAddress.new(address: "a@a.com"), contact_book: contact_book) } 116 | 117 | it "can handle queries" do 118 | contact_books = ContactBook.jit_preload.to_a 119 | expect(contact_books.first.companies_with_blank_email_address_count).to eq 1 120 | expect(contact_books.first.companies_with_blank_email_address).to eq [contact2] 121 | end 122 | end 123 | end 124 | 125 | context "when preloading an aggregate as polymorphic" do 126 | let(:contact_owner_counts) { [2] } 127 | 128 | context "without jit preload" do 129 | it "generates N+1 query notifications for each one" do 130 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 131 | ContactOwner.all.each_with_index do |c, i| 132 | expect(c.contacts_count).to eql contact_owner_counts[i] 133 | end 134 | end 135 | 136 | contact_owner_queries = [contact_owner].product([["contacts.count"]]) 137 | expect(source_map).to eql(Hash[contact_owner_queries]) 138 | end 139 | end 140 | 141 | context "with jit_preload" do 142 | 143 | it "does NOT generate N+1 query notifications" do 144 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 145 | ContactOwner.jit_preload.each_with_index do |c, i| 146 | expect(c.contacts_count).to eql contact_owner_counts[i] 147 | end 148 | end 149 | 150 | expect(source_map).to eql({}) 151 | end 152 | 153 | it "can handle queries" do 154 | ContactOwner.jit_preload.each_with_index do |c, i| 155 | expect(c.contacts_count).to eql contact_owner_counts[i] 156 | end 157 | end 158 | 159 | context "when a record has a polymorphic association type that's not an ActiveRecord" do 160 | before do 161 | contact1.update!(contact_owner_type: "NilClass", contact_owner_id: nil) 162 | end 163 | 164 | it "doesn't die while trying to load the association" do 165 | expect(Contact.jit_preload.map(&:contact_owner)).to eq [nil, ContactOwner.first, Address.first] 166 | end 167 | end 168 | 169 | context "when a record has a polymorphic association type is nil" do 170 | before do 171 | contact1.update!(contact_owner_type: nil, contact_owner_id: nil) 172 | end 173 | 174 | it "successfully load the rest of association values and does not publish a n+1 notification" do 175 | contacts = Contact.jit_preload.to_a 176 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 177 | expect(contacts.first.contact_owner).to eq(nil) 178 | end 179 | 180 | expect(source_map).to eql({}) 181 | 182 | expect do 183 | contacts.first.contact_owner 184 | contacts.second.contact_owner 185 | contacts.third.contact_owner 186 | end.not_to make_database_queries 187 | 188 | expect(contacts.second.contact_owner).to eq(ContactOwner.first) 189 | expect(contacts.third.contact_owner).to eq(Address.first) 190 | end 191 | end 192 | end 193 | end 194 | 195 | context "when preloading an aggregate on a has_many through relationship" do 196 | let(:country_contacts_counts) { [2, 3] } 197 | 198 | context "without jit preload" do 199 | it "generates N+1 query notifications for each one" do 200 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 201 | Country.all.each_with_index do |c, i| 202 | expect(c.contacts_count).to eql country_contacts_counts[i] 203 | end 204 | end 205 | 206 | country_contact_queries = [canada, usa].product([["contacts.count"]]) 207 | expect(source_map).to eql(Hash[country_contact_queries]) 208 | end 209 | end 210 | 211 | context "with jit_preload" do 212 | it "does NOT generate N+1 query notifications" do 213 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 214 | Country.all.jit_preload.each_with_index do |c, i| 215 | expect(c.contacts_count).to eql country_contacts_counts[i] 216 | end 217 | end 218 | 219 | expect(source_map).to eql({}) 220 | end 221 | 222 | it "can handle queries" do 223 | Country.all.jit_preload.each_with_index do |c, i| 224 | expect(c.contacts_count).to eql country_contacts_counts[i] 225 | end 226 | end 227 | end 228 | end 229 | 230 | context "when accessing an association with a scope that has a parameter" do 231 | let!(:contact_book) { ContactBook.create(name: "The Yellow Pages") } 232 | let!(:contact) { Contact.create(name: "Contact", contact_book: contact_book) } 233 | let!(:company1) { Company.create(name: "Company1", contact_book: contact_book) } 234 | 235 | it "is unable to be preloaded" do 236 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 237 | ContactBook.all.jit_preload.each do |contact_book| 238 | expect(contact_book.contacts_with_scope.to_a).to eql [company1, contact] 239 | end 240 | end 241 | 242 | expect(source_map).to eql(Hash[contact_book, [:contacts_with_scope]]) 243 | end 244 | end 245 | 246 | context "when preloading an aggregate on a polymorphic has_many through relationship" do 247 | let(:contact_owner_addresses_counts) { [3] } 248 | 249 | context "without jit preload" do 250 | it "generates N+1 query notifications for each one" do 251 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 252 | ContactOwner.all.each_with_index do |c, i| 253 | expect(c.addresses_count).to eql contact_owner_addresses_counts[i] 254 | end 255 | end 256 | 257 | contact_owner_addresses_queries = [contact_owner].product([["addresses.count"]]) 258 | expect(source_map).to eql(Hash[contact_owner_addresses_queries]) 259 | end 260 | end 261 | 262 | context "with jit_preload" do 263 | it "does NOT generate N+1 query notifications" do 264 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 265 | ContactOwner.all.jit_preload.each_with_index do |c, i| 266 | expect(c.addresses_count).to eql contact_owner_addresses_counts[i] 267 | end 268 | end 269 | 270 | expect(source_map).to eql({}) 271 | end 272 | 273 | it "can handle queries" do 274 | ContactOwner.all.jit_preload.each_with_index do |c, i| 275 | expect(c.addresses_count).to eql contact_owner_addresses_counts[i] 276 | end 277 | end 278 | end 279 | end 280 | 281 | context "when preloading a has_many through polymorphic aggregate where the through class has a polymorphic relationship to the target class" do 282 | let(:contact_owner_counts) { [1, 2] } 283 | 284 | context "without jit preload" do 285 | it "generates N+1 query notifications for each one" do 286 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 287 | Country.all.each_with_index do |c, i| 288 | expect(c.contact_owners_count).to eql contact_owner_counts[i] 289 | end 290 | end 291 | 292 | contact_owner_queries = [canada, usa].product([["contact_owners.count"]]) 293 | expect(source_map).to eql(Hash[contact_owner_queries]) 294 | end 295 | end 296 | 297 | context "with jit_preload" do 298 | it "does NOT generate N+1 query notifications" do 299 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 300 | Country.all.jit_preload.each_with_index do |c, i| 301 | expect(c.contact_owners_count).to eql contact_owner_counts[i] 302 | end 303 | end 304 | 305 | expect(source_map).to eql({}) 306 | end 307 | 308 | it "can handle queries" do 309 | Country.all.jit_preload.each_with_index do |c, i| 310 | expect(c.contact_owners_count).to eql contact_owner_counts[i] 311 | end 312 | end 313 | end 314 | end 315 | 316 | context "when a singular association id changes after preload" do 317 | let!(:contact_book1) { ContactBook.create(name: "The Yellow Pages") } 318 | let!(:contact_book2) { ContactBook.create(name: "The White Pages") } 319 | let!(:company1) { Company.create(name: "Company1", contact_book: contact_book1) } 320 | let!(:company2) { Company.create(name: "Company2", contact_book: contact_book1) } 321 | 322 | it "allows the association to be reloaded" do 323 | companies = Company.where(id: [company1.id, company2.id]).jit_preload.all.to_a 324 | expect(companies.map(&:contact_book)).to match_array [contact_book1, contact_book1] 325 | 326 | company = companies.each {|c| c.contact_book_id = contact_book2.id } 327 | 328 | expect(companies.map(&:contact_book)).to match_array [contact_book2, contact_book2] 329 | end 330 | end 331 | 332 | context "when preloading an aggregate" do 333 | let(:addresses_counts) { [3, 0, 2] } 334 | let(:phone_number_counts) { [2, 0, 1] } 335 | let(:maxes) { [19, 0, 12] } 336 | 337 | context "without jit_preload" do 338 | it "generates N+1 query notifications for each one" do 339 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 340 | Contact.all.each_with_index do |c, i| 341 | expect(c.addresses_count).to eql addresses_counts[i] 342 | expect(c.addresses_max_street_length).to eql maxes[i] 343 | expect(c.phone_numbers_count).to eql phone_number_counts[i] 344 | end 345 | end 346 | 347 | contact_queries = [contact1, contact2, contact3].product([["addresses.count", "addresses.maximum", "phone_numbers.count"]]) 348 | expect(source_map).to eql(Hash[contact_queries]) 349 | end 350 | end 351 | 352 | context "with jit_preload" do 353 | let(:usa_addresses_counts) { [2, 0, 1] } 354 | let(:can_addresses_counts) { [1, 0, 1] } 355 | 356 | it "does NOT generate N+1 query notifications" do 357 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 358 | Contact.jit_preload.each_with_index do |c, i| 359 | expect(c.addresses_count).to eql addresses_counts[i] 360 | expect(c.addresses_max_street_length).to eql maxes[i] 361 | expect(c.phone_numbers_count).to eql phone_number_counts[i] 362 | end 363 | end 364 | 365 | expect(source_map).to eql({}) 366 | end 367 | 368 | it "can handle dynamic queries" do 369 | Contact.jit_preload.each_with_index do |c, i| 370 | expect(c.addresses_count(country: usa)).to eql usa_addresses_counts[i] 371 | expect(c.addresses_count(country: canada)).to eql can_addresses_counts[i] 372 | end 373 | end 374 | end 375 | end 376 | 377 | context "when we marshal dump the active record object" do 378 | it "nullifes the jit_preloader reference" do 379 | contacts = Contact.jit_preload.to_a 380 | reloaded_contacts = contacts.collect{|r| Marshal.load(Marshal.dump(r)) } 381 | contacts.each do |c| 382 | expect(c.jit_preloader).to_not be_nil 383 | end 384 | reloaded_contacts.each do |c| 385 | expect(c.jit_preloader).to be_nil 386 | end 387 | end 388 | end 389 | 390 | context "when the preloader is globally enabled" do 391 | around do |example| 392 | JitPreloader.globally_enabled = true 393 | example.run 394 | JitPreloader.globally_enabled = false 395 | end 396 | it "doesn't reference the same records array that is returned" do 397 | contacts = Contact.all.to_a 398 | contacts << "A string" 399 | expect(contacts.first.jit_preloader.records).to eql Contact.all.to_a 400 | end 401 | 402 | context "when grabbing all of the address'es contries and email addresses" do 403 | it "doesn't generate an N+1 query ntoification" do 404 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 405 | Contact.all.collect{|c| c.addresses.collect(&:country); c.email_address } 406 | end 407 | expect(source_map).to eql({}) 408 | end 409 | end 410 | 411 | context "when we perform aggregate functions on the data" do 412 | it "generates N+1 query notifications for each one" do 413 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 414 | Contact.all.each{|c| c.addresses.count; c.addresses.sum(:id) } 415 | end 416 | contact_queries = [contact1,contact2, contact3].product([["addresses.count", "addresses.sum"]]) 417 | expect(source_map).to eql(Hash[contact_queries]) 418 | end 419 | end 420 | end 421 | 422 | context "when the preloader is not globally enabled" do 423 | context "when we perform aggregate functions on the data" do 424 | it "generates N+1 query notifications for each one" do 425 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 426 | Contact.all.each{|c| c.addresses.count; c.addresses.sum(:id) } 427 | end 428 | contact_queries = [contact1,contact2, contact3].product([["addresses.count", "addresses.sum"]]) 429 | expect(source_map).to eql(Hash[contact_queries]) 430 | end 431 | end 432 | 433 | context "when explicitly finding a contact" do 434 | it "generates N+1 query notifications for the country" do 435 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 436 | Contact.find(contact1.id).tap{|c| c.addresses.collect(&:country); c.email_address } 437 | end 438 | address_queries = Address.where(contact_id: 1).to_a.product([[:country]]) 439 | expect(source_map).to eql(Hash[address_queries]) 440 | end 441 | end 442 | 443 | context "when explicitly finding multiple contacts" do 444 | it "generates N+1 query notifications for the country" do 445 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 446 | Contact.find(contact1.id, contact2.id).each{|c| c.addresses.collect(&:country); c.email_address } 447 | end 448 | contact_queries = [contact1,contact2].product([[:addresses, :email_address]]) 449 | address_queries = Address.where(contact_id: contact1.id).to_a.product([[:country]]) 450 | 451 | expect(source_map).to eql(Hash[address_queries.concat(contact_queries)]) 452 | end 453 | end 454 | 455 | context "when grabbing the email address and address's country of the first contact" do 456 | it "generates N+1 query notifications for the country" do 457 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 458 | Contact.first.tap{|c| c.addresses.collect(&:country); c.email_address } 459 | end 460 | 461 | address_queries = Address.where(contact_id: contact1.id).to_a.product([[:country]]) 462 | 463 | expect(source_map).to eql(Hash[address_queries]) 464 | end 465 | end 466 | 467 | context "when grabbing all of the address'es contries and email addresses" do 468 | it "generates an N+1 query for each association on the contacts" do 469 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 470 | Contact.all.each{|c| c.addresses.collect(&:country); c.email_address } 471 | end 472 | contact_queries = [contact1,contact2,contact3].product([[:addresses, :email_address]]) 473 | address_queries = Address.all.to_a.product([[:country]]) 474 | expect(source_map).to eql(Hash[address_queries.concat(contact_queries)]) 475 | end 476 | 477 | context "and we use regular preload for addresses" do 478 | it "generates an N+1 query for only the email addresses on the contacts" do 479 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 480 | Contact.preload(:addresses).each{|c| c.addresses.collect(&:country); c.email_address } 481 | end 482 | contact_queries = [contact1,contact2,contact3].product([[:email_address]]) 483 | address_queries = Address.all.to_a.product([[:country]]) 484 | expect(source_map).to eql(Hash[address_queries.concat(contact_queries)]) 485 | end 486 | end 487 | 488 | context "and we use jit preload" do 489 | it "generates no n+1 queries" do 490 | ActiveSupport::Notifications.subscribed(callback, "n_plus_one_query") do 491 | Contact.jit_preload.each{|c| c.addresses.collect(&:country); c.email_address } 492 | end 493 | expect(source_map).to eql({}) 494 | end 495 | end 496 | 497 | context "reload" do 498 | it "clears the jit_preload_aggregates" do 499 | contact = Contact.jit_preload.first 500 | 501 | contact.addresses_count 502 | 503 | expect { contact.reload }.to change{ contact.jit_preload_aggregates }.to({}) 504 | end 505 | end 506 | end 507 | 508 | context "with dive limit set" do 509 | let!(:contact_book_1) { ContactBook.create(name: "The Yellow Pages") } 510 | let!(:contact_book_2) { ContactBook.create(name: "The Yellow Pages") } 511 | let!(:contact_book_3) { ContactBook.create(name: "The Yellow Pages") } 512 | let!(:company1) { Company.create(name: "Company1", contact_book: contact_book_1) } 513 | let!(:company2) { Company.create(name: "Company2", contact_book: contact_book_1) } 514 | let!(:company3) { Company.create(name: "Company2", contact_book: contact_book_2) } 515 | let!(:company4) { Company.create(name: "Company4", contact_book: contact_book_3) } 516 | let!(:company5) { Company.create(name: "Company5", contact_book: contact_book_3) } 517 | 518 | context "from the global value" do 519 | before do 520 | JitPreloader.max_ids_per_query = 2 521 | end 522 | 523 | after do 524 | JitPreloader.max_ids_per_query = nil 525 | end 526 | 527 | it "can handle queries" do 528 | contact_books = ContactBook.jit_preload.to_a 529 | 530 | expect(contact_books.first.companies_count).to eq 2 531 | expect(contact_books.second.companies_count).to eq 1 532 | expect(contact_books.last.companies_count).to eq 2 533 | end 534 | 535 | it "makes the right number of queries based on dive limit" do 536 | contact_books = ContactBook.jit_preload.to_a 537 | expect do 538 | contact_books.first.companies_count 539 | end.to make_database_queries(count: 2) 540 | 541 | expect do 542 | contact_books.second.companies_count 543 | contact_books.last.companies_count 544 | end.to_not make_database_queries 545 | end 546 | end 547 | 548 | context "from aggregate argument" do 549 | it "can handle queries" do 550 | contact_books = ContactBook.jit_preload.to_a 551 | 552 | expect(contact_books.first.companies_count_with_max_ids_set).to eq 2 553 | expect(contact_books.second.companies_count_with_max_ids_set).to eq 1 554 | expect(contact_books.last.companies_count_with_max_ids_set).to eq 2 555 | end 556 | 557 | it "makes the right number of queries based on dive limit" do 558 | contact_books = ContactBook.jit_preload.to_a 559 | expect do 560 | contact_books.first.companies_count_with_max_ids_set 561 | end.to make_database_queries(count: 2) 562 | 563 | expect do 564 | contact_books.second.companies_count_with_max_ids_set 565 | contact_books.last.companies_count_with_max_ids_set 566 | end.to_not make_database_queries 567 | end 568 | end 569 | end 570 | end 571 | 572 | end 573 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | Bundler.setup 3 | 4 | require 'jit_preloader' 5 | require 'support/database' 6 | require 'support/models' 7 | require 'byebug' 8 | require 'database_cleaner' 9 | 10 | DatabaseCleaner.strategy = :transaction 11 | 12 | RSpec.configure do |config| 13 | config.before(:suite) do 14 | Database.connect! 15 | Database.build! 16 | end 17 | config.before do 18 | DatabaseCleaner.start 19 | end 20 | 21 | config.after do 22 | DatabaseCleaner.clean 23 | end 24 | 25 | # rspec-expectations config goes here. You can use an alternate 26 | # assertion/expectation library such as wrong or the stdlib/minitest 27 | # assertions if you prefer. 28 | config.expect_with :rspec do |expectations| 29 | # This option will default to `true` in RSpec 4. It makes the `description` 30 | # and `failure_message` of custom matchers include text for helper methods 31 | # defined using `chain`, e.g.: 32 | # be_bigger_than(2).and_smaller_than(4).description 33 | # # => "be bigger than 2 and smaller than 4" 34 | # ...rather than: 35 | # # => "be bigger than 2" 36 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 37 | end 38 | 39 | # rspec-mocks config goes here. You can use an alternate test double 40 | # library (such as bogus or mocha) by changing the `mock_with` option here. 41 | config.mock_with :rspec do |mocks| 42 | # Prevents you from mocking or stubbing a method that does not exist on 43 | # a real object. This is generally recommended, and will default to 44 | # `true` in RSpec 4. 45 | mocks.verify_partial_doubles = true 46 | end 47 | 48 | config.disable_monkey_patching! 49 | 50 | config.order = :random 51 | end 52 | -------------------------------------------------------------------------------- /spec/support/database.rb: -------------------------------------------------------------------------------- 1 | class Database 2 | def self.tables 3 | [ 4 | "CREATE TABLE contact_books (id INTEGER NOT NULL PRIMARY KEY, name VARCHAR(255))", 5 | "CREATE TABLE contacts (id INTEGER NOT NULL PRIMARY KEY, type VARCHAR(255), contact_book_id INTEGER, contact_id INTEGER, name VARCHAR(255), contact_owner_id INTEGER, contact_owner_type VARCHAR(255))", 6 | "CREATE TABLE contact_owners (id INTEGER NOT NULL PRIMARY KEY, name VARCHAR(255))", 7 | "CREATE TABLE addresses (id INTEGER NOT NULL PRIMARY KEY, contact_id INTEGER NOT NULL, country_id INTEGER NOT NULL, street VARCHAR(255))", 8 | "CREATE TABLE email_addresses (id INTEGER NOT NULL PRIMARY KEY, contact_id INTEGER NOT NULL, address VARCHAR(255))", 9 | "CREATE TABLE phone_numbers (id INTEGER NOT NULL PRIMARY KEY, contact_id INTEGER NOT NULL, phone VARCHAR(10))", 10 | "CREATE TABLE countries (id INTEGER NOT NULL PRIMARY KEY, name VARCHAR(255))", 11 | "CREATE TABLE parents_children (id INTEGER NOT NULL PRIMARY KEY, parent_id INTEGER, child_id INTEGER)", 12 | ] 13 | end 14 | 15 | def self.build! 16 | tables.each do |table| 17 | ActiveRecord::Base.connection.execute(table) 18 | end 19 | end 20 | 21 | def self.connect! 22 | ActiveRecord::Base.establish_connection adapter: 'sqlite3', database: ":memory:" 23 | end 24 | 25 | end 26 | -------------------------------------------------------------------------------- /spec/support/models.rb: -------------------------------------------------------------------------------- 1 | class ContactBook < ActiveRecord::Base 2 | has_many :contacts 3 | has_many :contacts_with_scope, ->(_contact_book) { desc }, class_name: "Contact", foreign_key: :contact_book_id 4 | has_many :employees, through: :contacts 5 | 6 | has_many :companies 7 | has_many :company_employees, through: :companies, source: :employees 8 | 9 | has_many :parents 10 | has_many :children, through: :parents 11 | 12 | has_many_aggregate :companies, :count, :count, "*" 13 | has_many_aggregate :companies, :count_with_max_ids_set, :count, "*", max_ids_per_query: 2 14 | has_many_aggregate :employees, :count, :count, "*" 15 | has_many_aggregate :company_employees, :count, :count, "*" 16 | has_many_aggregate :children, :count, :count, "*" 17 | 18 | has_many :companies_with_blank_email_address, -> { joins(:email_address).where(email_addresses: { address: "" }) }, class_name: "Company" 19 | has_many_aggregate :companies_with_blank_email_address, :count, :count, "*", table_alias_name: "contacts" 20 | end 21 | 22 | class Contact < ActiveRecord::Base 23 | belongs_to :contact_book 24 | belongs_to :contact_owner, polymorphic: true 25 | 26 | has_many :addresses 27 | has_many :phone_numbers 28 | has_one :email_address 29 | has_many :employees 30 | 31 | has_many_aggregate :addresses, :max_street_length, :maximum, "LENGTH(street)" 32 | has_many_aggregate :phone_numbers, :count, :count, "id" 33 | has_many_aggregate :addresses, :count, :count, "*" 34 | 35 | scope :desc, ->{ order(id: :desc) } 36 | end 37 | 38 | class Company < Contact 39 | end 40 | 41 | class Employee < Contact 42 | belongs_to :contact 43 | end 44 | 45 | class ParentsChild < ActiveRecord::Base 46 | belongs_to :parent 47 | belongs_to :child 48 | end 49 | 50 | class Parent < Contact 51 | has_many :parents_child 52 | has_many :children, through: :parents_child 53 | end 54 | 55 | class Child < Contact 56 | has_many :parents_child 57 | has_many :parents, through: :parents_child 58 | end 59 | 60 | class Address < ActiveRecord::Base 61 | belongs_to :contact 62 | belongs_to :country 63 | end 64 | 65 | class EmailAddress < ActiveRecord::Base 66 | belongs_to :contact 67 | end 68 | 69 | class PhoneNumber < ActiveRecord::Base 70 | belongs_to :contact 71 | end 72 | 73 | class Country < ActiveRecord::Base 74 | has_many :addresses 75 | has_many :contacts, through: :addresses 76 | has_many :contact_owners, through: :contacts, source_type: 'ContactOwner' 77 | 78 | has_many_aggregate :contacts, :count, :count, "*" 79 | has_many_aggregate :contact_owners, :count, :count, "*" 80 | end 81 | 82 | class ContactOwner < ActiveRecord::Base 83 | has_many :contacts, as: :contact_owner 84 | has_many :addresses, through: :contacts 85 | 86 | has_many_aggregate :contacts, :count, :count, "*" 87 | has_many_aggregate :addresses, :count, :count, "*" 88 | end 89 | --------------------------------------------------------------------------------