├── .rspec ├── Gemfile ├── .env.example ├── lib ├── dblint │ ├── version.rb │ ├── checks │ │ ├── base.rb │ │ ├── missing_index.rb │ │ └── long_held_lock.rb │ └── rails_integration.rb └── dblint.rb ├── spec ├── support │ ├── application.rb │ ├── models.rb │ └── schema.rb ├── spec_helper.rb └── lib │ ├── missing_index_spec.rb │ ├── long_held_lock_spec.rb │ └── rails_integration_spec.rb ├── .gitignore ├── gemfiles ├── rails_4.2.gemfile └── rails_5.0.gemfile ├── Rakefile ├── CHANGELOG.md ├── .rubocop.yml ├── .travis.yml ├── LICENSE.txt ├── dblint.gemspec ├── CODE_OF_CONDUCT.md ├── Gemfile.lock └── README.md /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | DATABASE_URL=postgresql://user@localhost:5432/dblint 2 | -------------------------------------------------------------------------------- /lib/dblint/version.rb: -------------------------------------------------------------------------------- 1 | module Dblint 2 | VERSION = '0.2.0'.freeze 3 | end 4 | -------------------------------------------------------------------------------- /spec/support/application.rb: -------------------------------------------------------------------------------- 1 | class TestApplication < Rails::Application 2 | end 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /.env 4 | /Gemfile.lock 5 | /_yardoc/ 6 | /coverage/ 7 | /doc/ 8 | /pkg/ 9 | /spec/reports/ 10 | /tmp/ 11 | -------------------------------------------------------------------------------- /gemfiles/rails_4.2.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem 'activerecord', '~> 4.2' 4 | gem 'railties', '~> 4.2' 5 | 6 | group :development do 7 | gem 'rubocop', '0.39' 8 | end 9 | 10 | gemspec :path => "../" 11 | -------------------------------------------------------------------------------- /gemfiles/rails_5.0.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem 'activerecord', '~> 5.0' 4 | gem 'railties', '~> 5.0' 5 | 6 | group :development do 7 | gem 'rubocop', '0.39' 8 | end 9 | 10 | gemspec :path => "../" 11 | -------------------------------------------------------------------------------- /lib/dblint.rb: -------------------------------------------------------------------------------- 1 | require 'active_record' 2 | 3 | require 'dblint/version' 4 | 5 | require 'dblint/checks/base' 6 | require 'dblint/checks/long_held_lock' 7 | require 'dblint/checks/missing_index' 8 | 9 | require 'dblint/rails_integration' 10 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 2 | 3 | require 'dblint' 4 | require 'dotenv' 5 | require 'rails' 6 | 7 | Dotenv.load 8 | 9 | Dir[File.expand_path('../support/**/*.rb', __FILE__)].each { |f| require f } 10 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | require 'rspec/core/rake_task' 3 | require 'rubocop/rake_task' 4 | require 'dotenv/tasks' 5 | 6 | RSpec::Core::RakeTask.new 7 | RuboCop::RakeTask.new 8 | 9 | task default: [:lint, :spec] 10 | task test: :spec 11 | task lint: :rubocop 12 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.2.0 2016-07-29 4 | 5 | * Rails 5 support [#2](https://github.com/lfittl/dblint/pull/2) [@ksykulev](https://github.com/ksykulev) 6 | 7 | 8 | ## 0.1.1 2016-04-27 9 | 10 | * Fix handling of vendored gems [#1](https://github.com/lfittl/dblint/pull/1) [@ksykulev](https://github.com/ksykulev) 11 | 12 | 13 | ## 0.1.0 2015-05-04 14 | 15 | * Initial release 16 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | require: rubocop-rspec 2 | 3 | AllCops: 4 | Exclude: 5 | - spec/**/* 6 | - vendor/**/* 7 | 8 | Documentation: 9 | Enabled: false 10 | 11 | LineLength: 12 | Enabled: false 13 | 14 | MethodLength: 15 | Enabled: false 16 | 17 | ClassLength: 18 | Enabled: false 19 | 20 | Metrics/BlockNesting: 21 | Enabled: false 22 | 23 | Metrics/AbcSize: 24 | Enabled: false 25 | 26 | Style/ModuleFunction: 27 | Enabled: false 28 | -------------------------------------------------------------------------------- /spec/support/models.rb: -------------------------------------------------------------------------------- 1 | class Minion < ActiveRecord::Base 2 | has_many :dodos, counter_cache: true 3 | end 4 | 5 | class Dodo < ActiveRecord::Base 6 | belongs_to :minion, touch: true 7 | belongs_to :kitten 8 | end 9 | 10 | class Kitten < ActiveRecord::Base 11 | has_many :dodos 12 | has_many :minions, through: :dodos 13 | end 14 | 15 | RSpec.configure do |config| 16 | config.after(:suite) do 17 | # :sad_panda: 18 | Minion.delete_all 19 | Dodo.delete_all 20 | Kitten.delete_all 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/lib/missing_index_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Dblint::Checks::MissingIndex do 4 | it 'does not raise an error on non-filtering SELECTs' do 5 | expect do 6 | Minion.count 7 | end.not_to raise_error 8 | end 9 | 10 | it 'does not raise an error on filtering SELECTs with an index' do 11 | Minion.create! ident: 'test' 12 | expect do 13 | Minion.find_by!(ident: 'test') 14 | end.not_to raise_error 15 | end 16 | 17 | it 'does raise an error on filtering SELECTs without an index' do 18 | Minion.create! name: 'test' 19 | expect do 20 | Minion.find_by!(name: 'test') 21 | end.to raise_error(Dblint::Checks::MissingIndex::Error, /Missing index on/) 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/support/schema.rb: -------------------------------------------------------------------------------- 1 | RSpec.configure do |config| 2 | config.before(:suite) do 3 | ActiveRecord::Base.establish_connection ENV['DATABASE_URL'] 4 | 5 | ActiveRecord::Schema.define do 6 | self.verbose = false 7 | 8 | create_table :minions, force: true do |t| 9 | t.string :name 10 | t.string :ident, index: true 11 | t.timestamps null: false 12 | end 13 | 14 | create_table :dodos, force: true do |t| 15 | t.references :minion 16 | t.references :kitten 17 | t.timestamps null: false 18 | end 19 | 20 | create_table :kittens, force: true do |t| 21 | t.string :name 22 | t.timestamps null: false 23 | end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - rbx-3.29 4 | - 1.9.3-p551 5 | - 2.0.0-p648 6 | - 2.1.8 7 | - 2.2.4 8 | - 2.3.1 9 | 10 | gemfile: 11 | - gemfiles/rails_4.2.gemfile 12 | - gemfiles/rails_5.0.gemfile 13 | 14 | matrix: 15 | exclude: 16 | - gemfile: gemfiles/rails_5.0.gemfile 17 | rvm: 1.9.3-p551 18 | - gemfile: gemfiles/rails_5.0.gemfile 19 | rvm: 2.0.0-p648 20 | - gemfile: gemfiles/rails_5.0.gemfile 21 | rvm: 2.1.8 22 | - gemfile: gemfiles/rails_5.0.gemfile 23 | rvm: rbx-3.29 24 | 25 | addons: 26 | postgresql: 9.4 27 | env: 28 | - DATABASE_URL=postgresql://postgres@localhost:5432/dblint 29 | before_install: 30 | - gem install bundler 31 | before_script: 32 | - createdb dblint 33 | script: bundle exec rake 34 | -------------------------------------------------------------------------------- /lib/dblint/checks/base.rb: -------------------------------------------------------------------------------- 1 | require 'active_support/core_ext/string/inflections' 2 | 3 | module Dblint 4 | module Checks 5 | class Base 6 | def check_name 7 | self.class.name.demodulize 8 | end 9 | 10 | def find_main_app_caller(callstack) 11 | main_app_caller = callstack.find { |f| f.start_with?(Rails.root.to_s) && !f.include?('/vendor/bundle') } 12 | main_app_caller.slice!(Rails.root.to_s + '/') 13 | 14 | main_app_dir = main_app_caller[/^\w+/] 15 | return if %w(spec test).include?(main_app_dir) 16 | 17 | main_app_caller 18 | end 19 | 20 | def ignored?(main_app_caller) 21 | return false unless config && config['IgnoreList'] 22 | 23 | ignores = config['IgnoreList'][check_name] 24 | ignores.present? && ignores.include?(main_app_caller) 25 | end 26 | 27 | def config 28 | @config ||= begin 29 | config_file = Rails.root.join('.dblint.yml') 30 | YAML.load(File.read(config_file)) if File.exist?(config_file) 31 | end 32 | end 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Lukas Fittl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /dblint.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'dblint/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = 'dblint' 8 | spec.version = Dblint::VERSION 9 | spec.authors = ['Lukas Fittl'] 10 | spec.email = ['lukas@fittl.com'] 11 | 12 | spec.summary = 'Automatically tests your Rails app for common database query mistakes' 13 | spec.homepage = 'https://github.com/lfittl/dblint' 14 | spec.license = 'MIT' 15 | 16 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 17 | spec.require_paths = ['lib'] 18 | 19 | spec.add_runtime_dependency 'activerecord', '>= 4.0' 20 | spec.add_runtime_dependency 'railties', '>= 4.0' 21 | spec.add_runtime_dependency 'pg' 22 | 23 | spec.add_development_dependency 'bundler', '~> 1.9' 24 | spec.add_development_dependency 'rake', '~> 10.0' 25 | spec.add_development_dependency 'rspec', '~> 3' 26 | spec.add_development_dependency 'rspec-rails' 27 | spec.add_development_dependency 'sqlite3' 28 | spec.add_development_dependency 'dotenv' 29 | spec.add_development_dependency 'rubocop', '~> 0.39' 30 | spec.add_development_dependency 'rubocop-rspec' 31 | end 32 | -------------------------------------------------------------------------------- /spec/lib/long_held_lock_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Dblint::Checks::LongHeldLock do 4 | it 'does not raise an error if no UPDATE' do 5 | expect do 6 | Minion.transaction do 7 | 16.times { Kitten.count } 8 | end 9 | end.not_to raise_error 10 | end 11 | 12 | it 'does not raise an error if 15 or less transactions happened after an UPDATE' do 13 | minion = Minion.create! name: 'test' 14 | expect do 15 | Minion.transaction do 16 | 8.times { Kitten.count } 17 | minion.update! name: 'something' 18 | 8.times { Kitten.count } 19 | end 20 | end.not_to raise_error 21 | end 22 | 23 | it 'does not raise an error if the record was created in the transaction' do 24 | expect do 25 | Minion.transaction do 26 | minion2 = Minion.create! name: 'test' 27 | minion2.update! name: 'something' 28 | 16.times { Kitten.count } 29 | end 30 | end.not_to raise_error 31 | end 32 | 33 | it 'raises an error if more than 15 transactions happened after an UPDATE' do 34 | minion = Minion.create! name: 'test' 35 | expect do 36 | Minion.transaction do 37 | minion.update! name: 'something' 38 | 16.times { Kitten.count } 39 | end 40 | end.to raise_error(Dblint::Checks::LongHeldLock::Error, /Lock on \["minions", \d+\] held for 16 statements/) 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities. 4 | 5 | We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, age, or religion. 6 | 7 | Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct. 8 | 9 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team. 10 | 11 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers. 12 | 13 | This Code of Conduct is adapted from the [Contributor Covenant](http:contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/) 14 | -------------------------------------------------------------------------------- /lib/dblint/rails_integration.rb: -------------------------------------------------------------------------------- 1 | require 'active_support/notifications' 2 | 3 | module Dblint 4 | class RailsIntegration 5 | CHECKS = [ 6 | Dblint::Checks::LongHeldLock, 7 | Dblint::Checks::MissingIndex 8 | ].freeze 9 | 10 | def check_instance_for_connection(connid, klass) 11 | @checks ||= {} 12 | @checks[connid] ||= {} 13 | @checks[connid][klass] ||= klass.new 14 | end 15 | 16 | def start(name, id, payload) 17 | return if %w(CACHE DBLINT SCHEMA).include?(payload[:name]) 18 | 19 | CHECKS.each do |check_klass| 20 | check = check_instance_for_connection(payload[:connection_id], check_klass) 21 | check.statement_started(name, id, payload) 22 | end 23 | end 24 | 25 | def finish(name, id, payload) 26 | return if %w(CACHE DBLINT SCHEMA).include?(payload[:name]) 27 | 28 | CHECKS.each do |check_klass| 29 | check = check_instance_for_connection(payload[:connection_id], check_klass) 30 | check.statement_finished(name, id, payload_from_version(ActiveRecord::VERSION::MAJOR, payload)) 31 | end 32 | end 33 | 34 | private 35 | 36 | def payload_from_version(version, payload) 37 | return payload if version == 4 38 | 39 | compatible_binds = payload[:binds].map { |bind| [bind, bind.value] } 40 | payload.merge(binds: compatible_binds) 41 | end 42 | end 43 | 44 | ActiveSupport::Notifications.subscribe('sql.active_record', RailsIntegration.new) 45 | end 46 | -------------------------------------------------------------------------------- /lib/dblint/checks/missing_index.rb: -------------------------------------------------------------------------------- 1 | module Dblint 2 | module Checks 3 | class MissingIndex < Base 4 | class Error < StandardError; end 5 | 6 | def statement_started(_name, _id, payload) 7 | return if payload[:sql].include?(';') 8 | return unless payload[:sql].starts_with?('SELECT') 9 | 10 | ActiveRecord::Base.connection.execute 'SET enable_seqscan = off', 'DBLINT' 11 | plan = explain(payload) 12 | raise_on_seqscan(plan[0]['Plan'], payload) 13 | ActiveRecord::Base.connection.execute 'SET enable_seqscan = on', 'DBLINT' 14 | end 15 | 16 | def statement_finished(_name, _id, _payload) 17 | # Ignored 18 | end 19 | 20 | private 21 | 22 | def explain(payload) 23 | plan = ActiveRecord::Base.connection.select_value(format('EXPLAIN (FORMAT JSON) %s', payload[:sql]), 24 | 'DBLINT', payload[:binds]) 25 | JSON.parse(plan) 26 | end 27 | 28 | def raise_on_seqscan(plan, payload) 29 | if plan['Node Type'] == 'Seq Scan' && plan['Filter'].present? 30 | main_app_caller = find_main_app_caller(caller) 31 | return unless main_app_caller.present? 32 | 33 | return if ignored?(main_app_caller) 34 | 35 | error_msg = format("Missing index on %s for '%s' in '%s', called by %s", 36 | plan['Relation Name'], plan['Filter'], payload[:sql], main_app_caller) 37 | raise Error, error_msg 38 | end 39 | 40 | (plan['Plans'] || []).each do |subplan| 41 | raise_on_seqscan(subplan, payload) 42 | end 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /spec/lib/rails_integration_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | RSpec.describe Dblint::RailsIntegration do 4 | subject { Dblint::RailsIntegration.new } 5 | 6 | describe 'private#payload_from_version' do 7 | describe 'AR version 4', skip: ActiveRecord::VERSION::MAJOR != 4 do 8 | let(:payload) { 9 | { 10 | sql: 'SELECT "users".* FROM "users" WHERE "users"."id" = $1 AND "users"."visibility" = $2 ORDER BY "users"."id" ASC LIMIT 1', 11 | name: 'User Load', 12 | connection_id: 70108286766780, 13 | statement_name: nil, 14 | binds: [ 15 | [ActiveRecord::ConnectionAdapters::PostgreSQLColumn.new( 16 | 'id', 17 | nil, 18 | ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Uuid.new, 19 | 'uuid', 20 | false, 21 | 'uuid_generate_v4()'), 22 | '3b12f5f9-3d6c-4fd0-ba18-dc02a2b631af'], 23 | [ActiveRecord::ConnectionAdapters::PostgreSQLColumn.new( 24 | 'visibility', 25 | 'public', 26 | ActiveRecord::Type::String.new, 27 | 'character varying', 28 | false), 29 | 'public'] 30 | ] 31 | } 32 | } 33 | 34 | it 'returns payload' do 35 | expect(subject.send(:payload_from_version, 4, payload)).to eq(payload) 36 | end 37 | end 38 | 39 | describe "AR version 5", skip: ActiveRecord::VERSION::MAJOR != 5 do 40 | let(:payload) { 41 | { 42 | sql: 'SELECT "users".* FROM "users" WHERE "users"."id" = $1 AND "users"."visibility" = $2 ORDER BY "users"."id" ASC LIMIT 1', 43 | name: 'User Load', 44 | connection_id: 70108286766780, 45 | statement_name: nil, 46 | binds: [ 47 | ActiveRecord::Relation::QueryAttribute.new( 48 | 'id', 49 | '3b12f5f9-3d6c-4fd0-ba18-dc02a2b631af', 50 | ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Uuid.new 51 | ), 52 | ActiveRecord::Relation::QueryAttribute.new( 53 | 'visibility', 54 | 'public', 55 | ActiveModel::Type::String.new 56 | ) 57 | ] 58 | } 59 | } 60 | 61 | it 'creates backwards compatible version of binds' do 62 | result = subject.send(:payload_from_version, 5, payload) 63 | id = result[:binds].first 64 | visibility = result[:binds].second 65 | 66 | expect(id[0].name).to eq('id') 67 | expect(id[1]).to eq('3b12f5f9-3d6c-4fd0-ba18-dc02a2b631af') 68 | 69 | expect(visibility[0].name).to eq('visibility') 70 | expect(visibility[1]).to eq('public') 71 | end 72 | 73 | it 'preserves base payload attributes' do 74 | result = subject.send(:payload_from_version, 5, payload) 75 | 76 | expect(result[:sql]).to eq(payload[:sql]) 77 | expect(result[:name]).to eq(payload[:name]) 78 | expect(result[:conection_id]).to eq(payload[:conection_id]) 79 | expect(result[:statement_name]).to eq(payload[:statement_name]) 80 | end 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /lib/dblint/checks/long_held_lock.rb: -------------------------------------------------------------------------------- 1 | module Dblint 2 | module Checks 3 | class LongHeldLock < Base 4 | class Error < StandardError; end 5 | 6 | def initialize 7 | @locks_held = {} 8 | @existing_ids = {} 9 | end 10 | 11 | def statement_started(_name, _id, _payload) 12 | # Ignored 13 | end 14 | 15 | def statement_finished(_name, _id, payload) 16 | if payload[:sql] == 'BEGIN' 17 | handle_begin 18 | elsif payload[:sql] == 'COMMIT' 19 | handle_commit 20 | elsif payload[:sql] == 'ROLLBACK' 21 | # do nothing 22 | elsif @existing_ids.present? 23 | increment_locks_held 24 | add_new_locks_held(payload) 25 | end 26 | end 27 | 28 | private 29 | 30 | def increment_locks_held 31 | @locks_held.each do |_, lock| 32 | lock[:count] += 1 33 | end 34 | end 35 | 36 | def add_new_locks_held(payload) 37 | locked_table = payload[:sql].match(/^UPDATE\s+"?([\w.]+)"?/i).try(:[], 1) 38 | 39 | return unless locked_table.present? 40 | 41 | bind = payload[:binds].find { |b| b[0].name == 'id' } 42 | return unless bind.present? 43 | 44 | tuple = [locked_table, bind[1]] 45 | 46 | # We only want tuples that were not created in this transaction 47 | existing_ids = @existing_ids[tuple[0]] 48 | return unless existing_ids.present? && existing_ids.include?(tuple[1]) 49 | 50 | # We've done two UPDATEs to the same row in this transaction 51 | return if @locks_held[tuple].present? 52 | 53 | @locks_held[tuple] = {} 54 | @locks_held[tuple][:sql] = payload[:sql] 55 | @locks_held[tuple][:count] = 0 56 | @locks_held[tuple][:started_at] = Time.now 57 | end 58 | 59 | def handle_begin 60 | @locks_held = {} 61 | @existing_ids = {} 62 | 63 | ActiveRecord::Base.connection.tables.each do |table| 64 | next if %w(schema_migrations ar_internal_metadata).include?(table) 65 | @existing_ids[table] = ActiveRecord::Base.connection.select_values("SELECT id FROM #{table}", 'DBLINT').map(&:to_i) 66 | end 67 | end 68 | 69 | def handle_commit 70 | @locks_held.each do |table, details| 71 | next if details[:count] < 10 72 | 73 | main_app_caller = find_main_app_caller(caller) 74 | next unless main_app_caller.present? 75 | 76 | next if ignored?(main_app_caller) 77 | 78 | error_msg = format("Lock on %s held for %d statements (%0.2f ms) by '%s', transaction started by %s", 79 | table.inspect, details[:count], Time.now - details[:started_at], details[:sql], 80 | main_app_caller) 81 | 82 | next unless details[:count] > 15 83 | 84 | # We need an explicit begin here since we're interrupting the transaction flow 85 | ActiveRecord::Base.connection.execute('BEGIN') 86 | raise Error, error_msg 87 | 88 | # TODO: Add a config setting for enabling this as a warning 89 | # puts format('Warning: %s', error_msg) 90 | end 91 | end 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | dblint (0.2.0) 5 | activerecord (>= 4.0) 6 | pg 7 | railties (>= 4.0) 8 | 9 | GEM 10 | remote: https://rubygems.org/ 11 | specs: 12 | actionpack (4.2.6) 13 | actionview (= 4.2.6) 14 | activesupport (= 4.2.6) 15 | rack (~> 1.6) 16 | rack-test (~> 0.6.2) 17 | rails-dom-testing (~> 1.0, >= 1.0.5) 18 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 19 | actionview (4.2.6) 20 | activesupport (= 4.2.6) 21 | builder (~> 3.1) 22 | erubis (~> 2.7.0) 23 | rails-dom-testing (~> 1.0, >= 1.0.5) 24 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 25 | activemodel (4.2.6) 26 | activesupport (= 4.2.6) 27 | builder (~> 3.1) 28 | activerecord (4.2.6) 29 | activemodel (= 4.2.6) 30 | activesupport (= 4.2.6) 31 | arel (~> 6.0) 32 | activesupport (4.2.6) 33 | i18n (~> 0.7) 34 | json (~> 1.7, >= 1.7.7) 35 | minitest (~> 5.1) 36 | thread_safe (~> 0.3, >= 0.3.4) 37 | tzinfo (~> 1.1) 38 | arel (6.0.3) 39 | ast (2.2.0) 40 | builder (3.2.2) 41 | diff-lcs (1.2.5) 42 | dotenv (2.1.1) 43 | erubis (2.7.0) 44 | i18n (0.7.0) 45 | json (1.8.3) 46 | loofah (2.0.3) 47 | nokogiri (>= 1.5.9) 48 | mini_portile2 (2.0.0) 49 | minitest (5.8.4) 50 | nokogiri (1.6.7.2) 51 | mini_portile2 (~> 2.0.0.rc2) 52 | parser (2.3.1.0) 53 | ast (~> 2.2) 54 | pg (0.18.4) 55 | powerpack (0.1.1) 56 | rack (1.6.4) 57 | rack-test (0.6.3) 58 | rack (>= 1.0) 59 | rails-deprecated_sanitizer (1.0.3) 60 | activesupport (>= 4.2.0.alpha) 61 | rails-dom-testing (1.0.7) 62 | activesupport (>= 4.2.0.beta, < 5.0) 63 | nokogiri (~> 1.6.0) 64 | rails-deprecated_sanitizer (>= 1.0.1) 65 | rails-html-sanitizer (1.0.3) 66 | loofah (~> 2.0) 67 | railties (4.2.6) 68 | actionpack (= 4.2.6) 69 | activesupport (= 4.2.6) 70 | rake (>= 0.8.7) 71 | thor (>= 0.18.1, < 2.0) 72 | rainbow (2.1.0) 73 | rake (10.5.0) 74 | rspec (3.4.0) 75 | rspec-core (~> 3.4.0) 76 | rspec-expectations (~> 3.4.0) 77 | rspec-mocks (~> 3.4.0) 78 | rspec-core (3.4.4) 79 | rspec-support (~> 3.4.0) 80 | rspec-expectations (3.4.0) 81 | diff-lcs (>= 1.2.0, < 2.0) 82 | rspec-support (~> 3.4.0) 83 | rspec-mocks (3.4.1) 84 | diff-lcs (>= 1.2.0, < 2.0) 85 | rspec-support (~> 3.4.0) 86 | rspec-rails (3.4.2) 87 | actionpack (>= 3.0, < 4.3) 88 | activesupport (>= 3.0, < 4.3) 89 | railties (>= 3.0, < 4.3) 90 | rspec-core (~> 3.4.0) 91 | rspec-expectations (~> 3.4.0) 92 | rspec-mocks (~> 3.4.0) 93 | rspec-support (~> 3.4.0) 94 | rspec-support (3.4.1) 95 | rubocop (0.39.0) 96 | parser (>= 2.3.0.7, < 3.0) 97 | powerpack (~> 0.1) 98 | rainbow (>= 1.99.1, < 3.0) 99 | ruby-progressbar (~> 1.7) 100 | unicode-display_width (~> 1.0, >= 1.0.1) 101 | rubocop-rspec (1.4.1) 102 | rubocop (= 0.39.0) 103 | ruby-progressbar (1.8.0) 104 | sqlite3 (1.3.11) 105 | thor (0.19.1) 106 | thread_safe (0.3.5) 107 | tzinfo (1.2.2) 108 | thread_safe (~> 0.1) 109 | unicode-display_width (1.0.3) 110 | 111 | PLATFORMS 112 | ruby 113 | 114 | DEPENDENCIES 115 | bundler (~> 1.9) 116 | dblint! 117 | dotenv 118 | rake (~> 10.0) 119 | rspec (~> 3) 120 | rspec-rails 121 | rubocop (~> 0.39) 122 | rubocop-rspec 123 | sqlite3 124 | 125 | BUNDLED WITH 126 | 1.11.2 127 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dblint [ ![](https://img.shields.io/gem/v/dblint.svg)](https://rubygems.org/gems/dblint) [ ![](https://img.shields.io/gem/dt/dblint.svg)](https://rubygems.org/gems/dblint) [ ![](https://travis-ci.org/lfittl/dblint.svg?branch=master)](https://travis-ci.org/lfittl/dblint) 2 | 3 | Automatically checks all SQL queries that are executed during your tests, to find common mistakes, including missing indices and locking issues due to long transactions. 4 | 5 | 6 | ## Installation 7 | 8 | Add this line to your application's Gemfile: 9 | 10 | ```ruby 11 | gem 'dblint', group: :test 12 | ``` 13 | 14 | And then execute: 15 | 16 | $ bundle 17 | 18 | ## Usage 19 | 20 | Run your tests as usual, `dblint` will raise an exception or output a warning should it detect a problem. 21 | 22 | Note that it will check the callstack for the problematic query, and only count it as an error if the callstack first line from within your app is not the `test` or `spec` directory. Therefore, should you use non-optimized code directly in your tests (e.g. as part of a factory), `dblint` will not raise an error. 23 | 24 | ## Missing indices 25 | 26 | `dblint` will find SELECT queries that might be missing an index: 27 | 28 | ``` 29 | Failures: 30 | 31 | 1) FeedsController#show 32 | Failure/Error: get :show, format: :atom 33 | Dblint::Checks::MissingIndex::Error: 34 | Missing index on oauth_applications for '((twitter_app_name)::text = 'My Feed App'::text)' in 'SELECT "oauth_applications".* FROM "oauth_applications" WHERE "oauth_applications"."twitter_app_name" = $1 LIMIT 1', called by app/controllers/feeds_controller.rb:6:in `show' 35 | # ./app/controllers/feeds_controller.rb:6:in `show' 36 | # ./spec/controllers/feeds_controller_spec.rb:12:in `block (3 levels) in ' 37 | # ./spec/spec_helper.rb:78:in `block (2 levels) in ' 38 | ``` 39 | 40 | This is done by `EXPLAIN`ing every SELECT statement run during your tests, and checking whether the execution plan contains a Sequential Scan that is filtered by a condition. Thus, if you were to do a count of all records, this check would not trigger. 41 | 42 | Nonetheless, there might still be false positives or simply cases where you don't want an index - use the ignore list in those cases. 43 | 44 | Note: `EXPLAIN` is run with `enable_seqscan = off` in order to avoid seeing a false positive Sequential Scan on indexed, but small tables (likely to happen with tests). 45 | 46 | ## Long held locks 47 | 48 | `dblint` will find long held locks in database transactions, causing delays and possibly deadlocks: 49 | 50 | ``` 51 | 1) Invites::AcceptInvite test 52 | Failure/Error: described_class.call(user: invited_user) 53 | Dblint::LongHeldLock: 54 | Lock on ["users", 3] held for 29 statements (0.13 ms) by 'UPDATE "users" SET "invited_by_id" = $1, "role" = $2, "updated_at" = $3 WHERE "users"."id" = $4', transaction started by app/services/invites/accept_invite.rb:20:in `run' 55 | # ./app/services/invites/accept_invite.rb:20:in `run' 56 | # ./app/services/invites/accept_invite.rb:7:in `call' 57 | # ./spec/services/invites/accept_invite_spec.rb:8:in `accept_invite' 58 | # ./spec/services/invites/accept_invite_spec.rb:64:in `block (3 levels) in ' 59 | # ./spec/spec_helper.rb:78:in `block (2 levels) in ' 60 | ``` 61 | 62 | In this case it means that the `users` row has been locked for 29 statements (which took `0.13ms` in test, but this would be much more on any cloud setup), which can lead to lock contention issues on the `users` table, assuming other transactions are also updating that same row. 63 | 64 | The correct fix for this depends on whether this is in user written code (i.e. a manual `ActiveRecord::Base.transaction` call), or caused by Rails built-ins like `touch: true` and `counter_cache: true`. In general you want to move that `UPDATE` to happen towards the end of the transaction, or move it completely outside (if you don't need the atomicity guarantee of the transaction). 65 | 66 | Note: Right now the lock check can't detect possible problems caused by non-DB activities (e.g. updating your search index inside the transaction). 67 | 68 | ## Ignoring false positives 69 | 70 | Since in some cases there might be a valid reason to not have an index, or to hold a lock for longer, 71 | you can add ignores to the `.dblint.yml` file like this: 72 | 73 | ``` 74 | IgnoreList: 75 | MissingIndex: 76 | # Explain why you ignore it 77 | - app/models/my_model.rb:20:in `load' 78 | LongHeldLock: 79 | # Explain why you ignore it 80 | - app/models/my_model.rb:20:in `load' 81 | ``` 82 | 83 | The line you need to add is the first caller in the callstack from your main 84 | application, also included in the error message for the check. 85 | 86 | ## Contributing 87 | 88 | 1. Fork it ( https://github.com/lfittl/dblint/fork ) 89 | 2. Create your feature branch (`git checkout -b my-new-feature`) 90 | 3. Commit your changes (`git commit -am 'Add some feature'`) 91 | 4. Push to the branch (`git push origin my-new-feature`) 92 | 5. Create a new Pull Request 93 | 94 | 95 | ## Authors 96 | 97 | - [Lukas Fittl](mailto:lukas@fittl.com) 98 | 99 | 100 | ## License 101 | 102 | Copyright (c) 2015, Lukas Fittl
103 | dblint is licensed under the MIT license, see LICENSE.txt file for details. 104 | --------------------------------------------------------------------------------