├── .rspec ├── bin └── setup ├── lib ├── io_monitor │ ├── version.rb │ ├── patches │ │ ├── abstract_adapter_patch.rb │ │ ├── net_http_adapter_patch.rb │ │ ├── future_result_patch.rb │ │ ├── redis_patch.rb │ │ └── action_controller_base_patch.rb │ ├── adapters │ │ ├── base_adapter.rb │ │ ├── redis_adapter.rb │ │ ├── net_http_adapter.rb │ │ └── active_record_adapter.rb │ ├── publishers │ │ ├── logs_publisher.rb │ │ ├── notifications_publisher.rb │ │ ├── base_publisher.rb │ │ └── prometheus_publisher.rb │ ├── railtie.rb │ ├── aggregator.rb │ ├── controller.rb │ └── configuration.rb └── io_monitor.rb ├── .standard.yml ├── spec ├── dummy │ ├── config │ │ ├── environment.rb │ │ ├── routes.rb │ │ ├── environments │ │ │ └── test.rb │ │ ├── boot.rb │ │ ├── database.yml │ │ ├── initializers │ │ │ └── io_to_response_payload_ratio.rb │ │ └── application.rb │ └── app │ │ ├── controllers │ │ ├── application_controller.rb │ │ └── fake_controller.rb │ │ └── models │ │ └── application_record.rb ├── io_monitor │ ├── publishers │ │ ├── logs_publisher_spec.rb │ │ ├── notifications_publisher_spec.rb │ │ ├── prometheus_publisher_spec.rb │ │ └── shared_examples.rb │ ├── adapters │ │ ├── net_http_adapter_spec.rb │ │ ├── active_record_adapter_spec.rb │ │ └── redis_adapter_spec.rb │ ├── aggregator_spec.rb │ ├── configuration_spec.rb │ └── controller_spec.rb ├── io_monitor_spec.rb └── spec_helper.rb ├── Rakefile ├── .gitignore ├── Gemfile ├── gemfiles ├── rails_7_0.gemfile ├── rails_7_1.gemfile ├── rails_7_2.gemfile ├── rails_8_0.gemfile └── railsmaster.gemfile ├── .github └── workflows │ ├── lint.yml │ └── test.yml ├── LICENSE.txt ├── io_monitor.gemspec ├── CHANGELOG.md └── README.md /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | -------------------------------------------------------------------------------- /lib/io_monitor/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | VERSION = "1.1.0" 5 | end 6 | -------------------------------------------------------------------------------- /.standard.yml: -------------------------------------------------------------------------------- 1 | # For available configuration options, see: 2 | # https://github.com/testdouble/standard 3 | ruby_version: 3.0 4 | -------------------------------------------------------------------------------- /spec/dummy/config/environment.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "application" 4 | 5 | Rails.application.initialize! 6 | -------------------------------------------------------------------------------- /spec/dummy/config/routes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Rails.application.routes.draw do 4 | get "/fake", to: "fake#fake" 5 | end 6 | -------------------------------------------------------------------------------- /spec/dummy/config/environments/test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Rails.application.configure do 4 | config.eager_load = false 5 | end 6 | -------------------------------------------------------------------------------- /spec/dummy/app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class ApplicationController < ActionController::API 4 | end 5 | -------------------------------------------------------------------------------- /spec/dummy/app/models/application_record.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class ApplicationRecord < ActiveRecord::Base 4 | primary_abstract_class 5 | end 6 | -------------------------------------------------------------------------------- /spec/dummy/config/boot.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile", __dir__) 4 | 5 | require "bundler/setup" 6 | -------------------------------------------------------------------------------- /spec/dummy/config/database.yml: -------------------------------------------------------------------------------- 1 | default: &default 2 | adapter: sqlite3 3 | pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> 4 | timeout: 5000 5 | 6 | test: 7 | <<: *default 8 | database: db/test.sqlite3 9 | -------------------------------------------------------------------------------- /spec/dummy/config/initializers/io_to_response_payload_ratio.rb: -------------------------------------------------------------------------------- 1 | IoMonitor.configure do |config| 2 | # Enable all available adapters for testing purposes. 3 | config.adapters = IoMonitor::ADAPTERS.map(&:new) 4 | end 5 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/gem_tasks" 4 | require "rspec/core/rake_task" 5 | require "standard/rake" 6 | 7 | RSpec::Core::RakeTask.new(:spec) 8 | 9 | task default: %i[spec standard] 10 | -------------------------------------------------------------------------------- /spec/dummy/app/controllers/fake_controller.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class FakeController < ApplicationController 4 | include IoMonitor::Controller 5 | 6 | def fake 7 | raise ActionController::RoutingError.new("Fake") 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/io_monitor/patches/abstract_adapter_patch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | module AbstractAdapterPatch 5 | def build_result(*args, **kwargs, &block) 6 | ActiveRecordAdapter.aggregate_result rows: kwargs[:rows] 7 | 8 | super 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | 10 | spec/dummy/db/*.sqlite3 11 | spec/dummy/db/*.sqlite3-journal 12 | spec/dummy/log/*.log 13 | spec/dummy/tmp/ 14 | 15 | .rspec_status 16 | 17 | Gemfile.lock 18 | .ruby-version 19 | .ruby-gemset 20 | .tool-versions -------------------------------------------------------------------------------- /lib/io_monitor/adapters/base_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | class BaseAdapter 5 | # :nocov: 6 | def self.kind 7 | raise NotImplementedError 8 | end 9 | 10 | def initialize! 11 | raise NotImplementedError 12 | end 13 | # :nocov: 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/io_monitor/adapters/redis_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "io_monitor/patches/redis_patch" 4 | 5 | module IoMonitor 6 | class RedisAdapter < BaseAdapter 7 | def self.kind 8 | :redis 9 | end 10 | 11 | def initialize! 12 | ActiveSupport.on_load(:after_initialize) do 13 | Redis.prepend(RedisPatch) 14 | end 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/io_monitor/patches/net_http_adapter_patch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | module NetHttpAdapterPatch 5 | def request(*args, &block) 6 | super.tap do |response| 7 | if response&.body && IoMonitor.aggregator.active? 8 | IoMonitor.aggregator.increment(NetHttpAdapter.kind, response.body.bytesize) 9 | end 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "rake", "~> 13.0" 4 | gem "rspec", "~> 3.0" 5 | gem "rspec-rails", "~> 5.0" 6 | gem "with_model", "~> 2.0" 7 | gem "database_cleaner-active_record", "~> 2.0" 8 | gem "standard", "~> 1.18.0" 9 | gem "simplecov", "~> 0.21.0" 10 | gem "pry" 11 | gem "webmock", "~> 3.14" 12 | gem "concurrent-ruby", "1.3.4" 13 | gem "sqlite3", "~> 1.4.0" 14 | gem "rails", "~> 7.1.0" 15 | 16 | gemspec 17 | -------------------------------------------------------------------------------- /lib/io_monitor/patches/future_result_patch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | module FutureResultPatch 5 | def result 6 | # @event_buffer is used to send ActiveSupport notifications related to async queries 7 | return super unless @event_buffer 8 | 9 | res = super 10 | ActiveRecordAdapter.aggregate_result rows: res.rows 11 | 12 | res 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /gemfiles/rails_7_0.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "rake", "~> 13.0" 4 | gem "rspec", "~> 3.0" 5 | gem "rspec-rails", "~> 5.0" 6 | gem "with_model", "~> 2.0" 7 | gem "database_cleaner-active_record", "~> 2.0" 8 | gem "standard", "~> 1.18.0" 9 | gem "simplecov", "~> 0.21.0" 10 | gem "pry" 11 | gem "webmock", "~> 3.14" 12 | gem "concurrent-ruby", "1.3.4" 13 | gem "sqlite3", "~> 1.4.0" 14 | gem "rails", "~> 7.0.0" 15 | 16 | gemspec path: "../" 17 | -------------------------------------------------------------------------------- /gemfiles/rails_7_1.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "rake", "~> 13.0" 4 | gem "rspec", "~> 3.0" 5 | gem "rspec-rails", "~> 5.0" 6 | gem "with_model", "~> 2.0" 7 | gem "database_cleaner-active_record", "~> 2.0" 8 | gem "standard", "~> 1.18.0" 9 | gem "simplecov", "~> 0.21.0" 10 | gem "pry" 11 | gem "webmock", "~> 3.14" 12 | gem "concurrent-ruby", "1.3.4" 13 | gem "sqlite3", "~> 1.4.0" 14 | gem "rails", "~> 7.1.0" 15 | 16 | gemspec path: "../" 17 | -------------------------------------------------------------------------------- /gemfiles/rails_7_2.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "rake", "~> 13.0" 4 | gem "rspec", "~> 3.0" 5 | gem "rspec-rails", "~> 5.0" 6 | gem "with_model", "~> 2.0" 7 | gem "database_cleaner-active_record", "~> 2.0" 8 | gem "standard", "~> 1.18.0" 9 | gem "simplecov", "~> 0.21.0" 10 | gem "pry" 11 | gem "webmock", "~> 3.14" 12 | gem "concurrent-ruby", "1.3.4" 13 | gem "sqlite3", "~> 1.4.0" 14 | gem "rails", "~> 7.2.0" 15 | 16 | gemspec path: "../" 17 | -------------------------------------------------------------------------------- /gemfiles/rails_8_0.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "rake", "~> 13.0" 4 | gem "rspec", "~> 3.0" 5 | gem "rspec-rails", "~> 5.0" 6 | gem "with_model", "~> 2.0" 7 | gem "database_cleaner-active_record", "~> 2.0" 8 | gem "standard", "~> 1.18.0" 9 | gem "simplecov", "~> 0.21.0" 10 | gem "pry" 11 | gem "webmock", "~> 3.14" 12 | gem "concurrent-ruby", "1.3.4" 13 | gem "sqlite3", "~> 2.1.0" 14 | gem "rails", "~> 8.0.0" 15 | 16 | gemspec path: "../" 17 | -------------------------------------------------------------------------------- /lib/io_monitor/publishers/logs_publisher.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | class LogsPublisher < BasePublisher 5 | def self.kind 6 | :logs 7 | end 8 | 9 | def publish(source, ratio) 10 | Rails.logger.warn <<~HEREDOC.squish 11 | #{source.to_s.camelize} I/O to response payload ratio is #{ratio}, 12 | while threshold is #{IoMonitor.config.warn_threshold} 13 | HEREDOC 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /gemfiles/railsmaster.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "rake", "~> 13.0" 4 | gem "rspec", "~> 3.0" 5 | gem "rspec-rails", "~> 5.0" 6 | gem "with_model", "~> 2.0" 7 | gem "database_cleaner-active_record", "~> 2.0" 8 | gem "standard", "~> 1.18.0" 9 | gem "simplecov", "~> 0.21.0" 10 | gem "pry" 11 | gem "webmock", "~> 3.14" 12 | gem "concurrent-ruby", "1.3.4" 13 | gem "sqlite3", "~> 2.1.0" 14 | gem "rails", branch: "main", git: "https://github.com/rails/rails.git" 15 | 16 | gemspec path: "../" 17 | -------------------------------------------------------------------------------- /spec/io_monitor/publishers/logs_publisher_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "shared_examples" 4 | 5 | RSpec.describe IoMonitor::LogsPublisher do 6 | subject(:publisher) { described_class.new } 7 | 8 | it_behaves_like "publisher" 9 | 10 | describe ".publish" do 11 | it "logs a warning" do 12 | msg = /ActiveRecord I\/O to response payload ratio is 0.5/ 13 | expect(Rails.logger).to receive(:warn).with(msg) 14 | 15 | publisher.publish(:active_record, 0.5) 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/io_monitor/patches/redis_patch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | module RedisPatch 5 | def send_command(command, &block) 6 | super(command, &block).tap do |reply| 7 | # we need to check QUEUED because of https://github.com/redis/redis-rb/blob/cbdb53e8c2f0be53c91404cb7ff566a36fc8ebf5/lib/redis/client.rb#L164 8 | if reply != "QUEUED" && !reply.is_a?(Redis::CommandError) && IoMonitor.aggregator.active? 9 | IoMonitor.aggregator.increment(RedisAdapter.kind, reply.bytesize) 10 | end 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/io_monitor/patches/action_controller_base_patch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | module ActionControllerBasePatch 5 | def log_process_action(payload) 6 | super.tap do |messages| 7 | next unless IoMonitor.config.publishers.any? { |publisher| publisher.is_a?(LogsPublisher) } 8 | 9 | data = payload[IoMonitor::NAMESPACE] 10 | next unless data 11 | 12 | data.each do |source, bytes| 13 | size = ActiveSupport::NumberHelper.number_to_human_size(bytes) 14 | messages << "#{source.to_s.camelize} Payload: #{size}" 15 | end 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/io_monitor/publishers/notifications_publisher.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | class NotificationsPublisher < BasePublisher 5 | WARN_THRESHOLD_REACHED_EVENT = "warn_threshold_reached" 6 | 7 | def self.kind 8 | :notifications 9 | end 10 | 11 | def publish(source, ratio) 12 | ActiveSupport::Notifications.instrument( 13 | full_event_name(WARN_THRESHOLD_REACHED_EVENT), 14 | source: source, 15 | ratio: ratio 16 | ) 17 | end 18 | 19 | private 20 | 21 | def full_event_name(event_name) 22 | "#{event_name}.#{IoMonitor::NAMESPACE}" 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/io_monitor/publishers/notifications_publisher_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "shared_examples" 4 | 5 | RSpec.describe IoMonitor::NotificationsPublisher do 6 | subject(:publisher) { described_class.new } 7 | 8 | it_behaves_like "publisher" 9 | 10 | describe ".publish" do 11 | it "instruments warn_threshold_reached event" do 12 | warn_event = "warn_threshold_reached.#{IoMonitor::NAMESPACE}" 13 | expect(ActiveSupport::Notifications).to receive(:instrument) 14 | .with(warn_event, source: :active_record, ratio: 0.5) 15 | 16 | publisher.publish(:active_record, 0.5) 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/io_monitor/adapters/net_http_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "net/http" 4 | require "io_monitor/patches/net_http_adapter_patch" 5 | 6 | module IoMonitor 7 | class NetHttpAdapter < BaseAdapter 8 | def self.kind 9 | :net_http 10 | end 11 | 12 | def initialize! 13 | ActiveSupport.on_load(:after_initialize) do 14 | Net::HTTP.prepend(NetHttpAdapterPatch) 15 | 16 | if defined?(::WebMock) 17 | WebMock::HttpLibAdapters::NetHttpAdapter 18 | .instance_variable_get(:@webMockNetHTTP) 19 | .prepend(NetHttpAdapterPatch) 20 | end 21 | end 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/io_monitor/railtie.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "io_monitor/patches/action_controller_base_patch" 4 | 5 | module IoMonitor 6 | class Railtie < Rails::Railtie 7 | config.after_initialize do 8 | IoMonitor.config.adapters.each(&:initialize!) 9 | 10 | ActiveSupport.on_load(:action_controller) do 11 | ActionController::Base.singleton_class.prepend(ActionControllerBasePatch) 12 | end 13 | 14 | ActiveSupport::Notifications.subscribe("process_action.action_controller") do |*args| 15 | payload = args.last[IoMonitor::NAMESPACE] 16 | next unless payload 17 | 18 | IoMonitor.config.publishers.each { |publisher| publisher.process_action(payload) } 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/dummy/config/application.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "boot" 4 | 5 | require "rails" 6 | require "active_model/railtie" 7 | require "active_record/railtie" 8 | require "action_controller/railtie" 9 | require "rails/test_unit/railtie" 10 | 11 | Bundler.require(*Rails.groups) 12 | 13 | module Dummy 14 | class Application < Rails::Application 15 | config.root = File.join(__dir__, "..") 16 | config.logger = Logger.new("/dev/null") 17 | config.api_only = true 18 | config.active_record.legacy_connection_handling = false if ActiveRecord::Base.respond_to?(:legacy_connection_handling=) 19 | 20 | if Rails::VERSION::MAJOR >= 7 21 | config.active_record.async_query_executor = :global_thread_pool 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/io_monitor/publishers/base_publisher.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | class BasePublisher 5 | # :nocov: 6 | def self.kind 7 | raise NotImplementedError 8 | end 9 | 10 | def publish(source, ratio) 11 | raise NotImplementedError 12 | end 13 | # :nocov: 14 | 15 | def process_action(payload) 16 | (payload.keys - [:response]).each do |source| 17 | ratio = ratio(payload[:response], payload[source]) 18 | 19 | publish(source, ratio) if ratio < IoMonitor.config.warn_threshold 20 | end 21 | end 22 | 23 | private 24 | 25 | def ratio(response_size, io_size) 26 | return 0 if io_size.to_f.zero? 27 | 28 | response_size.to_f / io_size.to_f 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /spec/io_monitor/publishers/prometheus_publisher_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "shared_examples" 4 | require "prometheus/client" 5 | 6 | RSpec.describe IoMonitor::PrometheusPublisher do 7 | subject(:publisher) { described_class.new(registry: registry) } 8 | 9 | let(:registry) { ::Prometheus::Client.registry } 10 | let(:metric) { ::Prometheus::Client::Histogram.new(:test, docstring: "...") } 11 | 12 | before { expect(registry).to receive(:histogram).and_return(metric) } 13 | 14 | it_behaves_like "publisher" 15 | 16 | describe ".publish" do 17 | it "changes prometheus metrics" do 18 | expect(metric).to receive(:observe) 19 | .with(0.5, labels: {adapter: :active_record}) 20 | 21 | publisher.publish(:active_record, 0.5) 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - '**' 8 | tags-ignore: 9 | - 'v*' 10 | 11 | jobs: 12 | rubocop: 13 | # Skip running tests for local pull requests (use push event instead), run only for foreign ones 14 | if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login 15 | name: Standard.rb 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Install SQLite 20 | run: sudo apt-get install libsqlite3-dev 21 | - uses: ruby/setup-ruby@v1 22 | with: 23 | ruby-version: "3.1" 24 | bundler-cache: true 25 | - name: Lint Ruby code with Standard.rb 26 | run: | 27 | bundle exec rake standard 28 | -------------------------------------------------------------------------------- /spec/io_monitor/adapters/net_http_adapter_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe IoMonitor::NetHttpAdapter do 4 | let(:aggregator) { IoMonitor.aggregator } 5 | let(:body) { "Hello, World!" } 6 | let(:url) { "www.example.com" } 7 | 8 | before do 9 | stub_request(:get, url).to_return body: body 10 | end 11 | 12 | around do |example| 13 | aggregator.collect { example.run } 14 | end 15 | 16 | context "when aggregator is inactive" do 17 | before do 18 | aggregator.stop! 19 | end 20 | 21 | it "does nothing" do 22 | expect(aggregator).not_to receive(:increment) 23 | 24 | Net::HTTP.get url, "/" 25 | end 26 | end 27 | 28 | it "increments aggregator by request's body bytesize" do 29 | allow(aggregator).to receive(:increment) 30 | 31 | Net::HTTP.get url, "/" 32 | 33 | expect(aggregator).to have_received(:increment).with(described_class.kind, body.bytesize) 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/io_monitor/aggregator.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | # Thread-safe payload size aggregator. 5 | class Aggregator 6 | def initialize(sources) 7 | @sources = sources 8 | end 9 | 10 | attr_reader :sources 11 | 12 | def active? 13 | InputPayload.active.present? 14 | end 15 | 16 | def collect 17 | start! 18 | yield 19 | stop! 20 | end 21 | 22 | def start! 23 | InputPayload.active = true 24 | end 25 | 26 | def stop! 27 | InputPayload.active = false 28 | end 29 | 30 | def increment(source, val) 31 | return unless active? 32 | 33 | InputPayload.state ||= empty_state 34 | InputPayload.state[source.to_sym] += val 35 | end 36 | 37 | def get(source) 38 | InputPayload.state ||= empty_state 39 | InputPayload.state[source.to_sym] 40 | end 41 | 42 | private 43 | 44 | def empty_state 45 | sources.map { |kind| [kind, 0] }.to_h 46 | end 47 | 48 | class InputPayload < ActiveSupport::CurrentAttributes 49 | attribute :state, :active 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2022 DmitryTsepelev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/io_monitor/adapters/active_record_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "io_monitor/patches/abstract_adapter_patch" 4 | require "io_monitor/patches/future_result_patch" 5 | 6 | module IoMonitor 7 | class ActiveRecordAdapter < BaseAdapter 8 | class << self 9 | def kind 10 | :active_record 11 | end 12 | 13 | def aggregate_result(rows:) 14 | return unless IoMonitor.aggregator.active? 15 | 16 | # `.flatten.join.bytesize` would look prettier, 17 | # but it makes a lot of unnecessary allocations. 18 | io_payload_size = rows.sum(0) do |row| 19 | row.sum(0) do |val| 20 | ((String === val) ? val : val.to_s).bytesize 21 | end 22 | end 23 | 24 | IoMonitor.aggregator.increment(kind, io_payload_size) 25 | end 26 | end 27 | 28 | def initialize! 29 | ActiveSupport.on_load(:active_record) do 30 | ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(AbstractAdapterPatch) 31 | 32 | if Rails::VERSION::MAJOR >= 7 33 | ActiveRecord::FutureResult.prepend(FutureResultPatch) 34 | end 35 | end 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /spec/io_monitor_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe IoMonitor do 4 | subject { described_class } 5 | 6 | it "has a version number" do 7 | expect(subject::VERSION).not_to be nil 8 | end 9 | 10 | describe ".configure" do 11 | it "yields config" do 12 | expect(subject).to receive(:configure).and_yield(subject.config) 13 | 14 | subject.configure { |config| } 15 | end 16 | end 17 | 18 | def without_memoization(object, property) 19 | ivar = "@#{property}".to_sym 20 | old_value = object.instance_variable_get(ivar) 21 | object.remove_instance_variable(ivar) 22 | 23 | yield 24 | 25 | object.instance_variable_set(ivar, old_value) 26 | end 27 | 28 | describe ".aggregator" do 29 | it "filters out disabled sources" do 30 | without_memoization(subject, :aggregator) do 31 | mock_adapter = Class.new do 32 | def self.kind 33 | :mock_adapter 34 | end 35 | end 36 | 37 | allow(subject.config).to receive(:adapters).and_return([mock_adapter.new]) 38 | 39 | expect(subject.aggregator.sources).to contain_exactly(:mock_adapter) 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "webmock/rspec" 4 | require "simplecov" 5 | SimpleCov.start if ENV["COVERAGE"] 6 | 7 | ENV["RAILS_ENV"] = "test" 8 | 9 | require "redis" 10 | require_relative "dummy/config/environment" 11 | 12 | require "rspec/rails" 13 | 14 | require "io_monitor" 15 | 16 | RSpec.configure do |config| 17 | # For proper work of ActiveSupport::CurrentAttributes reset 18 | config.include ActiveSupport::CurrentAttributes::TestHelper 19 | 20 | config.example_status_persistence_file_path = ".rspec_status" 21 | config.infer_base_class_for_anonymous_controllers = true 22 | 23 | config.extend WithModel 24 | 25 | if Rails::VERSION::MAJOR >= 7 26 | config.use_transactional_fixtures = true 27 | else 28 | config.before(:suite) do 29 | DatabaseCleaner.clean_with(:truncation) 30 | end 31 | 32 | config.before(:each) do |e| 33 | DatabaseCleaner.strategy = e.metadata[:skip_transaction] ? :truncation : :transaction 34 | DatabaseCleaner.start 35 | end 36 | 37 | config.append_after(:each) do 38 | DatabaseCleaner.clean 39 | end 40 | end 41 | 42 | config.expect_with :rspec do |c| 43 | c.syntax = :expect 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - '**' 8 | tags-ignore: 9 | - 'v*' 10 | 11 | jobs: 12 | test: 13 | name: 'Rails ${{ matrix.rails }} × Ruby ${{ matrix.ruby }}' 14 | # Skip running tests for local pull requests (use push event instead), run only for foreign ones 15 | if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login 16 | runs-on: ubuntu-latest 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | ruby: [3.1, 3.2, 3.3, 3.4] 21 | 22 | gemfile: [ 23 | "gemfiles/rails_7_0.gemfile", 24 | "gemfiles/rails_7_1.gemfile", 25 | "gemfiles/rails_7_2.gemfile", 26 | "gemfiles/rails_8_0.gemfile", 27 | "gemfiles/railsmaster.gemfile" 28 | ] 29 | 30 | allow_failures: 31 | - false 32 | steps: 33 | - uses: actions/checkout@v2 34 | - name: Install SQLite 35 | run: sudo apt-get install libsqlite3-dev 36 | - uses: ruby/setup-ruby@v1 37 | with: 38 | ruby-version: ${{ matrix.ruby }} 39 | bundler-cache: true 40 | - name: Run specs 41 | run: bundle exec rake spec 42 | -------------------------------------------------------------------------------- /lib/io_monitor/controller.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | module Controller 5 | extend ActiveSupport::Concern 6 | 7 | delegate :aggregator, to: IoMonitor 8 | 9 | ALL_ACTIONS = Object.new 10 | 11 | class_methods do 12 | def monitor_io_for(*actions_to_monitor_io) 13 | @actions_to_monitor_io = actions_to_monitor_io 14 | end 15 | 16 | def actions_to_monitor_io 17 | @actions_to_monitor_io || ALL_ACTIONS 18 | end 19 | end 20 | 21 | def process_action(*) 22 | if monitors_action?(action_name) 23 | aggregator.collect { super } 24 | else 25 | super 26 | end 27 | end 28 | 29 | def append_info_to_payload(payload) 30 | super 31 | 32 | return unless monitors_action?(action_name) 33 | 34 | data = payload[IoMonitor::NAMESPACE] = {} 35 | 36 | aggregator.sources.each do |source| 37 | data[source] = aggregator.get(source) 38 | end 39 | 40 | data[:response] = payload[:response]&.body&.bytesize || 0 41 | end 42 | 43 | private 44 | 45 | def monitors_action?(action_name) 46 | actions = self.class.actions_to_monitor_io 47 | actions == ALL_ACTIONS || actions.include?(action_name.to_sym) 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/io_monitor.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "io_monitor/version" 4 | require "io_monitor/configuration" 5 | require "io_monitor/aggregator" 6 | require "io_monitor/controller" 7 | 8 | require "io_monitor/adapters/base_adapter" 9 | require "io_monitor/adapters/active_record_adapter" 10 | require "io_monitor/adapters/net_http_adapter" 11 | 12 | require "io_monitor/publishers/base_publisher" 13 | require "io_monitor/publishers/logs_publisher" 14 | require "io_monitor/publishers/notifications_publisher" 15 | require "io_monitor/publishers/prometheus_publisher" 16 | 17 | require "io_monitor/railtie" 18 | 19 | module IoMonitor 20 | NAMESPACE = :io_monitor 21 | 22 | adapters = [ActiveRecordAdapter, NetHttpAdapter] 23 | 24 | if defined? Redis 25 | require "io_monitor/adapters/redis_adapter" 26 | adapters << RedisAdapter 27 | end 28 | ADAPTERS = adapters.freeze 29 | 30 | PUBLISHERS = [LogsPublisher, NotificationsPublisher, PrometheusPublisher].freeze 31 | 32 | class << self 33 | def aggregator 34 | @aggregator ||= Aggregator.new( 35 | config.adapters.map { |a| a.class.kind } 36 | ) 37 | end 38 | 39 | def config 40 | @config ||= Configuration.new 41 | end 42 | 43 | def configure 44 | yield config 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /io_monitor.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "lib/io_monitor/version" 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "io_monitor" 7 | spec.version = IoMonitor::VERSION 8 | spec.authors = ["baygeldin", "prog-supdex", "maxshend", "DmitryTsepelev"] 9 | spec.email = ["dmitry.a.tsepelev@gmail.com"] 10 | spec.homepage = "https://github.com/DmitryTsepelev/io_monitor" 11 | spec.summary = "A gem that helps to detect potential memory bloats" 12 | 13 | spec.license = "MIT" 14 | 15 | spec.metadata = { 16 | "bug_tracker_uri" => "https://github.com/DmitryTsepelev/io_monitor/issues", 17 | "changelog_uri" => "https://github.com/DmitryTsepelev/io_monitor/blob/master/CHANGELOG.md", 18 | "documentation_uri" => "https://github.com/DmitryTsepelev/io_monitor/blob/master/README.md", 19 | "homepage_uri" => "https://github.com/DmitryTsepelev/io_monitor", 20 | "source_code_uri" => "https://github.com/DmitryTsepelev/io_monitor" 21 | } 22 | 23 | spec.files = [ 24 | Dir.glob("lib/**/*"), 25 | "README.md", 26 | "CHANGELOG.md", 27 | "LICENSE.txt" 28 | ].flatten 29 | 30 | spec.require_paths = ["lib"] 31 | 32 | spec.required_ruby_version = ">= 3.1.0" 33 | 34 | spec.add_dependency "rails", ">= 7.0" 35 | spec.add_development_dependency "redis", ">= 4.0" 36 | spec.add_development_dependency "prometheus-client" 37 | end 38 | -------------------------------------------------------------------------------- /lib/io_monitor/publishers/prometheus_publisher.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | class PrometheusPublisher < BasePublisher 5 | HELP_MESSAGE = "IO payload size to response payload size ratio" 6 | 7 | def initialize(registry: nil, aggregation: nil, buckets: nil) 8 | registry ||= ::Prometheus::Client.registry 9 | @metric = registry.histogram( 10 | "#{IoMonitor::NAMESPACE}_ratio".to_sym, 11 | labels: %i[adapter], 12 | buckets: buckets || ::Prometheus::Client::Histogram::DEFAULT_BUCKETS, 13 | store_settings: store_settings(aggregation), 14 | docstring: HELP_MESSAGE 15 | ) 16 | end 17 | 18 | def self.kind 19 | :prometheus 20 | end 21 | 22 | def publish(source, ratio) 23 | metric.observe(ratio, labels: {adapter: source}) 24 | end 25 | 26 | private 27 | 28 | attr_reader :metric 29 | 30 | # From https://github.com/yabeda-rb/yabeda-prometheus/blob/v0.8.0/lib/yabeda/prometheus/adapter.rb#L101 31 | def store_settings(aggregation) 32 | case ::Prometheus::Client.config.data_store 33 | when ::Prometheus::Client::DataStores::Synchronized, ::Prometheus::Client::DataStores::SingleThreaded 34 | {} # Default synchronized store doesn't allow to pass any options 35 | when ::Prometheus::Client::DataStores::DirectFileStore, ::Object # Anything else 36 | {aggregation: aggregation}.compact 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /spec/io_monitor/adapters/active_record_adapter_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe IoMonitor::ActiveRecordAdapter do 4 | let(:aggregator) { IoMonitor.aggregator } 5 | 6 | with_model :Fake, scope: :all do 7 | table do |t| 8 | t.string :name 9 | end 10 | 11 | model {} 12 | end 13 | 14 | before(:all) do 15 | (1..10).each { |i| Fake.create!(name: "Fake #{i}") } 16 | end 17 | 18 | after(:all) do 19 | Fake.delete_all 20 | end 21 | 22 | around do |example| 23 | aggregator.collect { example.run } 24 | end 25 | 26 | describe "without async queries" do 27 | context "when aggregator is inactive" do 28 | before do 29 | aggregator.stop! 30 | end 31 | 32 | it "does nothing" do 33 | expect(aggregator).not_to receive(:increment) 34 | 35 | Fake.all.to_a 36 | end 37 | end 38 | 39 | it "increments aggregator by query result's bytesize" do 40 | allow(aggregator).to receive(:increment) 41 | 42 | bytesize = Fake.pluck(:id, :name).flatten.join.bytesize 43 | 44 | expect(aggregator).to have_received(:increment).with(described_class.kind, bytesize) 45 | end 46 | end 47 | 48 | if Rails::VERSION::MAJOR >= 7 49 | context "when load_async is used" do 50 | let!(:bytesize) { Fake.pluck(:id, :name).flatten.join.bytesize } 51 | 52 | it "increments aggregator by query result's bytesize", skip_transaction: true do 53 | allow(aggregator).to receive(:increment) 54 | 55 | Fake.all.load_async.then do 56 | expect(aggregator).to have_received(:increment).with(described_class.kind, bytesize) 57 | end 58 | end 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/io_monitor/configuration.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module IoMonitor 4 | class Configuration 5 | DEFAULT_WARN_THRESHOLD = 0.0 6 | 7 | def initialize 8 | @publishers = [LogsPublisher.new] 9 | @adapters = [ActiveRecordAdapter.new] 10 | @warn_threshold = DEFAULT_WARN_THRESHOLD 11 | end 12 | 13 | attr_reader :publishers, :adapters, :warn_threshold 14 | 15 | def publish=(values) 16 | @publishers = [*values].map { |value| value_to_publisher(value) } 17 | end 18 | 19 | def adapters=(value) 20 | @adapters = [*value].map do |adapter| 21 | if adapter.is_a?(BaseAdapter) 22 | adapter 23 | elsif (adapter_type = resolve(IoMonitor::ADAPTERS, adapter)) 24 | adapter_type.new 25 | else 26 | supported = IoMonitor::ADAPTERS.map(&:kind) 27 | raise ArgumentError, "Only the following adapters are supported: #{supported}." 28 | end 29 | end 30 | end 31 | 32 | def warn_threshold=(value) 33 | case value 34 | when 0..1 35 | @warn_threshold = value.to_f 36 | else 37 | raise ArgumentError, "Warn threshold should be within 0..1 range." 38 | end 39 | end 40 | 41 | private 42 | 43 | def resolve(list, kind) 44 | list.find { |p| p.kind == kind } 45 | end 46 | 47 | def value_to_publisher(value) 48 | if value.is_a?(BasePublisher) 49 | value 50 | elsif (publisher_type = resolve(IoMonitor::PUBLISHERS, value)) 51 | publisher_type.new 52 | else 53 | supported = IoMonitor::PUBLISHERS.map(&:kind) 54 | raise ArgumentError, "Only the following publishers are supported: #{supported}." 55 | end 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /spec/io_monitor/adapters/redis_adapter_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../../../lib/io_monitor/adapters/redis_adapter" 4 | 5 | RSpec.describe IoMonitor::RedisAdapter do 6 | let(:aggregator) { IoMonitor.aggregator } 7 | 8 | let(:key) { "key" } 9 | let(:value) { "Hello, World!" } 10 | 11 | let(:redis_client) { instance_double(Redis::Client) } 12 | let(:redis) { Redis.new } 13 | 14 | before do 15 | allow(redis_client).to receive(:call_v).and_return(value) 16 | allow(Redis::Client).to receive(:new).and_return(redis_client) 17 | end 18 | 19 | around do |example| 20 | aggregator.collect { example.run } 21 | end 22 | 23 | context "when aggregator is inactive" do 24 | before do 25 | aggregator.stop! 26 | end 27 | 28 | it "does nothing" do 29 | expect(aggregator).not_to receive(:increment) 30 | redis.get(key) 31 | end 32 | end 33 | 34 | it "increments aggregator by request's body bytesize" do 35 | allow(aggregator).to receive(:increment) 36 | redis.get(key) 37 | expect(aggregator).to have_received(:increment).with(described_class.kind, value.bytesize) 38 | end 39 | 40 | context "when response is QUEUED" do 41 | before do 42 | allow(redis_client).to receive(:call_v).and_return("QUEUED") 43 | end 44 | 45 | it "does nothing" do 46 | expect(aggregator).not_to receive(:increment) 47 | redis.get(key) 48 | end 49 | end 50 | 51 | context "when response is error" do 52 | before do 53 | allow(redis_client).to receive(:call_v).and_return(Redis::CommandError.new("ERROR")) 54 | end 55 | 56 | it "does nothing" do 57 | expect(aggregator).not_to receive(:increment) 58 | redis.get(key) 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change log 2 | 3 | ## main 4 | 5 | ## 1.1.0 (2025-01-21) 6 | 7 | - [PR#23](https://github.com/DmitryTsepelev/io_monitor/pull/23) Fix net/http adapter and an issue with nil responses ([@SSDany]) 8 | 9 | ## 1.0.0 (2023-05-06) 10 | 11 | - [PR#22](https://github.com/DmitryTsepelev/io_monitor/pull/22) Handle zero payload ([@DmitryTsepelev]) 12 | - [PR#17](https://github.com/DmitryTsepelev/io_monitor/pull/17) Prometheus publisher ([@maxshend]) 13 | - [PR#10](https://github.com/DmitryTsepelev/io_monitor/pull/10) Per–action monitoring ([@DmitryTsepelev]) 14 | - [PR#15](https://github.com/DmitryTsepelev/io_monitor/pull/15) Allow configure more than one publisher ([@DmitryTsepelev]) 15 | - [PR#9](https://github.com/DmitryTsepelev/io_monitor/pull/9) Restrict minimum Rails version to 6.1, adjust test matrix, and related changes ([@Envek]) 16 | 17 | ## 0.2.0 (2022-05-29) 18 | 19 | - [PR#8](https://github.com/DmitryTsepelev/io_monitor/pull/8) Add Redis adapter ([@DmitryTsepelev]) 20 | 21 | ## 0.1.0 (2022-05-24) 22 | 23 | - [PR#7](https://github.com/DmitryTsepelev/io_monitor/pull/7) Add HTTP adapter ([@maxshend]) 24 | - [PR#6](https://github.com/DmitryTsepelev/io_monitor/pull/6) Add support for ActiveRecord::Relation#load_async method ([@maxshend]) 25 | - [PR#5](https://github.com/DmitryTsepelev/io_monitor/pull/5) Use ActiveSupport::CurrentAttributes to store input payload ([@maxshend]) 26 | - [PR#2](https://github.com/DmitryTsepelev/io_monitor/pull/2), [PR#3](https://github.com/DmitryTsepelev/io_monitor/pull/3), [PR#4](https://github.com/DmitryTsepelev/io_monitor/pull/4) Initial implementation ([@prog-supdex], [@maxshend], [@baygeldin]) 27 | 28 | [@baygeldin]: https://github.com/baygeldin 29 | [@prog-supdex]: https://github.com/prog-supdex 30 | [@maxshend]: https://github.com/maxshend 31 | [@DmitryTsepelev]: https://github.com/DmitryTsepelev 32 | [@Envek]: https://github.com/Envek 33 | [@SSDany]: https://github.com/SSDany 34 | -------------------------------------------------------------------------------- /spec/io_monitor/aggregator_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe IoMonitor::Aggregator do 4 | subject(:aggregator) { described_class.new(sources) } 5 | 6 | let(:sources) { %i[active_record] } 7 | 8 | describe ".active?" do 9 | it "false by default" do 10 | expect(aggregator.active?).to be false 11 | end 12 | end 13 | 14 | describe ".start!" do 15 | it "makes it active" do 16 | expect { aggregator.start! }.to change { aggregator.active? }.from(false).to(true) 17 | end 18 | end 19 | 20 | describe ".stop!" do 21 | it "makes it inactive" do 22 | aggregator.start! 23 | 24 | expect { aggregator.stop! }.to change { aggregator.active? }.from(true).to(false) 25 | end 26 | end 27 | 28 | describe ".increment" do 29 | it "increments the specified source value" do 30 | aggregator.start! 31 | aggregator.increment(sources.first, 42) 32 | expect(aggregator.get(sources.first)).to eq(42) 33 | end 34 | 35 | context "when inactive" do 36 | it "doesn't increment the specified source value" do 37 | aggregator.increment(sources.first, 42) 38 | expect(aggregator.get(sources.first)).to eq(0) 39 | end 40 | end 41 | end 42 | 43 | describe ".get" do 44 | it "returns the specified source value" do 45 | aggregator.start! 46 | aggregator.increment(sources.first, 42) 47 | expect(aggregator.get(sources.first)).to eq(42) 48 | end 49 | end 50 | 51 | it "is thread-safe" do 52 | aggregator.start! 53 | aggregator.increment(sources.first, 42) 54 | aggregator.stop! 55 | 56 | Thread.new do 57 | aggregator.start! 58 | 59 | expect(aggregator.get(sources.first)).to eq(0) 60 | expect(aggregator.active?).to be true 61 | end.join 62 | 63 | expect(aggregator.get(sources.first)).to eq(42) 64 | expect(aggregator.active?).to be false 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /spec/io_monitor/publishers/shared_examples.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.shared_examples "publisher" do |parameter| 4 | describe ".process_action" do 5 | let(:event) { "process_action.action_controller" } 6 | let(:full_payload) { {IoMonitor::NAMESPACE => payload} } 7 | 8 | let(:payload) do 9 | { 10 | io_kind => io_payload_size, 11 | :response => response_payload_size 12 | } 13 | end 14 | 15 | let(:io_kind) { :active_record } 16 | let(:io_payload_size) { 42 } 17 | let(:response_payload_size) { 42 } 18 | 19 | let(:warn_threshold) { 0.0 } 20 | 21 | before do 22 | IoMonitor.configure do |config| 23 | config.warn_threshold = warn_threshold 24 | config.publish = publisher 25 | end 26 | end 27 | 28 | let(:ratio) { response_payload_size.to_f / io_payload_size.to_f } 29 | 30 | it "is called on process_action.action_controller event" do 31 | expect(publisher).to receive(:process_action).with(payload) 32 | 33 | ActiveSupport::Notifications.instrument(event, full_payload) 34 | end 35 | 36 | context "when event is irrelevant" do 37 | let(:full_payload) { {} } 38 | 39 | it "does nothing" do 40 | expect(publisher).not_to receive(:process_action) 41 | 42 | ActiveSupport::Notifications.instrument(event, full_payload) 43 | end 44 | end 45 | 46 | context "when warn threshold is reached" do 47 | let(:warn_threshold) { 0.8 } 48 | let(:io_payload_size) { response_payload_size * 10 } 49 | 50 | it "calls .publish method with source and ratio" do 51 | expect(publisher).to receive(:publish).with(io_kind, ratio) 52 | 53 | publisher.process_action(payload) 54 | end 55 | 56 | context "when io_payload_size is zero" do 57 | let(:io_payload_size) { 0 } 58 | let(:ratio) { 0 } 59 | 60 | it "calls .publish method with source and ratio" do 61 | expect(publisher).to receive(:publish).with(io_kind, ratio) 62 | 63 | publisher.process_action(payload) 64 | end 65 | end 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /spec/io_monitor/configuration_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe IoMonitor::Configuration do 4 | subject(:config) { described_class.new } 5 | 6 | describe ".publish" do 7 | it "resolves publisher by kind" do 8 | config.publish = :notifications 9 | expect(config.publishers.first).to be_a(IoMonitor::NotificationsPublisher) 10 | end 11 | 12 | context "when kind is unknown" do 13 | it "raises an error" do 14 | expect { config.publish = :whatever }.to raise_error(ArgumentError) 15 | end 16 | end 17 | 18 | it "allows custom publishers" do 19 | custom = Class.new(IoMonitor::BasePublisher) {} 20 | config.publish = custom.new 21 | expect(config.publishers.first).to be_a(custom) 22 | end 23 | 24 | it "equals to :logs by default" do 25 | expect(config.publishers.first).to be_a(IoMonitor::LogsPublisher) 26 | end 27 | 28 | context "when multiple publishers are set" do 29 | it "assigns all publishers" do 30 | config.publish = [:notifications, :logs] 31 | expect(config.publishers.first).to be_a(IoMonitor::NotificationsPublisher) 32 | expect(config.publishers.last).to be_a(IoMonitor::LogsPublisher) 33 | end 34 | end 35 | end 36 | 37 | describe ".adapters" do 38 | it "allows a single adapter instead of an array" do 39 | expect { config.adapters = :active_record }.not_to raise_error 40 | end 41 | 42 | it "resolves adapters by kind" do 43 | config.adapters = %i[active_record] 44 | expect(config.adapters.first).to be_a(IoMonitor::ActiveRecordAdapter) 45 | end 46 | 47 | context "when kind is unknown" do 48 | it "raises an error" do 49 | expect { config.adapters = %i[whatever] }.to raise_error(ArgumentError) 50 | end 51 | end 52 | 53 | it "allows custom adapters" do 54 | custom = Class.new(IoMonitor::BaseAdapter) {} 55 | config.adapters = [custom.new] 56 | expect(config.adapters.first).to be_a(custom) 57 | end 58 | 59 | it "equals to :active_record by default" do 60 | expect(config.adapters.first).to be_a(IoMonitor::ActiveRecordAdapter) 61 | end 62 | end 63 | 64 | describe ".warn_threshold" do 65 | it "allows only values in 0..1 range" do 66 | config.warn_threshold = 0.5 67 | expect(config.warn_threshold).to eq(0.5) 68 | 69 | expect { config.warn_threshold = 42 }.to raise_error(ArgumentError) 70 | end 71 | 72 | it "equals to zero by default" do 73 | expect(config.warn_threshold).to eq(0.0) 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /spec/io_monitor/controller_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe IoMonitor::Controller, type: :controller do 4 | controller(ApplicationController) do 5 | include IoMonitor::Controller 6 | 7 | def index 8 | render json: Fake.all 9 | end 10 | end 11 | 12 | with_model :Fake, scope: :all do 13 | table do |t| 14 | t.string :name 15 | end 16 | 17 | model {} 18 | end 19 | 20 | before(:all) do 21 | (1..10).each { |i| Fake.create!(name: "Fake #{i}") } 22 | end 23 | 24 | after(:all) do 25 | Fake.delete_all 26 | end 27 | 28 | before do 29 | IoMonitor.configure do |config| 30 | config.publish = publisher 31 | end 32 | end 33 | 34 | let(:publisher) { :logs } 35 | 36 | it "adds info to process_action.action_controller payload for each source" do 37 | event = "process_action.action_controller" 38 | 39 | subscription = ActiveSupport::Notifications.subscribe(event) do |*args| 40 | payload = args.last[IoMonitor::NAMESPACE] 41 | 42 | expect(payload[:active_record]).to eq(Fake.pluck(:id, :name).flatten.join.bytesize) 43 | expect(payload[:response]).to eq(args.last[:response].body.bytesize) 44 | end 45 | 46 | get :index 47 | 48 | ActiveSupport::Notifications.unsubscribe(subscription) 49 | end 50 | 51 | # ActionControllerBasePatch related specs are here as well for convenience: 52 | 53 | def get_info_logs 54 | [].tap do |infos| 55 | # `info` is called with a block in ActionController::LogSubscriber 56 | allow(Rails.logger).to receive(:info) { |&block| infos << block.call } 57 | 58 | yield 59 | end 60 | end 61 | 62 | context "when publisher is set to logs" do 63 | let(:publisher) { :logs } 64 | 65 | it "adds info to ActionController log entry" do 66 | infos = get_info_logs { get :index } 67 | 68 | expect(infos).to include(/Completed 200 OK/) 69 | expect(infos).to include(/ActiveRecord Payload: \d+ Bytes/) 70 | expect(infos).to include(/Response Payload: \d+ Bytes/) 71 | end 72 | end 73 | 74 | context "when publisher is not set to logs" do 75 | let(:publisher) { :notifications } 76 | 77 | it "doesn't modify ActionController log entry" do 78 | infos = get_info_logs { get :index } 79 | 80 | expect(infos).to include(/Completed 200 OK/) 81 | expect(infos).not_to include(/ActiveRecord Payload:/) 82 | expect(infos).not_to include(/Response Payload:/) 83 | end 84 | end 85 | 86 | context "when concern is not included" do 87 | controller(ApplicationController) do 88 | def index 89 | render json: Fake.all 90 | end 91 | end 92 | 93 | it "doesn't modify ActionController log entry" do 94 | infos = get_info_logs { get :index } 95 | 96 | expect(infos).to include(/Completed 200 OK/) 97 | expect(infos).not_to include(/ActiveRecord Payload:/) 98 | expect(infos).not_to include(/Response Payload:/) 99 | end 100 | end 101 | 102 | context "when monitor_io_for is configured" do 103 | controller(ApplicationController) do 104 | include IoMonitor::Controller 105 | 106 | monitor_io_for :index 107 | 108 | def index 109 | render json: Fake.all 110 | end 111 | 112 | def show 113 | render json: Fake.first 114 | end 115 | end 116 | 117 | it "adds info to ActionController log entry" do 118 | infos = get_info_logs { get :index } 119 | 120 | expect(infos).to include(/Completed 200 OK/) 121 | expect(infos).to include(/ActiveRecord Payload: \d+ Bytes/) 122 | expect(infos).to include(/Response Payload: \d+ Bytes/) 123 | end 124 | 125 | context "when action is not included to the list of monitored ones" do 126 | it "doesn't modify ActionController log entry" do 127 | infos = get_info_logs { get :show, params: {id: 1} } 128 | 129 | expect(infos).to include(/Completed 200 OK/) 130 | expect(infos).not_to include(/ActiveRecord Payload:/) 131 | expect(infos).not_to include(/Response Payload:/) 132 | end 133 | end 134 | end 135 | end 136 | 137 | RSpec.describe "when response is nil", type: :request do 138 | it "does not fail" do 139 | get "/fake" 140 | 141 | expect(response).to be_not_found 142 | end 143 | end 144 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # IoMonitor 2 | 3 | [![Gem Version](https://badge.fury.io/rb/io_monitor.svg)](https://rubygems.org/gems/io_monitor) 4 | [![Tests status](https://github.com/DmitryTsepelev/io_monitor/actions/workflows/test.yml/badge.svg)](https://github.com/DmitryTsepelev/io_monitor/actions/workflows/test.yml) 5 | ![](https://ruby-gem-downloads-badge.herokuapp.com/io_monitor?type=total) 6 | 7 | A gem that helps to detect potential memory bloats. 8 | 9 | When your controller loads a lot of data to the memory but returns a small response to the client it might mean that you're using the IO in the non–optimal way. In this case, you'll see the following message in your logs: 10 | 11 | ``` 12 | Completed 200 OK in 349ms (Views: 2.1ms | ActiveRecord: 38.7ms | ActiveRecord Payload: 866.00 B | Response Payload: 25.00 B | Allocations: 72304) 13 | ``` 14 | 15 | You can support my open–source work [here](https://boosty.to/dmitry_tsepelev). 16 | 17 | ## Usage 18 | 19 | Add this line to your application's Gemfile: 20 | 21 | ```ruby 22 | gem 'io_monitor' 23 | ``` 24 | 25 | Currently gem can collect the data from `ActiveRecord`, `Net::HTTP` and `Redis`. 26 | 27 | Change configuration in an initializer if you need: 28 | 29 | ```ruby 30 | IoMonitor.configure do |config| 31 | config.publish = [:logs, :notifications, :prometheus] # defaults to :logs 32 | config.warn_threshold = 0.8 # defaults to 0 33 | config.adapters = [:active_record, :net_http, :redis] # defaults to [:active_record] 34 | end 35 | ``` 36 | 37 | Then include the concern into your controller: 38 | 39 | ```ruby 40 | class MyController < ApplicationController 41 | include IoMonitor::Controller 42 | end 43 | ``` 44 | 45 | Depending on configuration when IO payload size to response payload size ratio reaches the threshold either a `warn_threshold_reached.io_monitor` notification will be sent or a following warning will be logged: 46 | 47 | ``` 48 | ActiveRecord I/O to response payload ratio is 0.1, while threshold is 0.8 49 | ``` 50 | Prometheus metrics example: 51 | ``` 52 | ... 53 | # TYPE io_monitor_ratio histogram 54 | # HELP io_monitor_ratio IO payload size to response payload size ratio 55 | io_monitor_ratio_bucket{adapter="active_record",le="0.01"} 0.0 56 | io_monitor_ratio_bucket{adapter="active_record",le="5"} 2.0 57 | io_monitor_ratio_bucket{adapter="active_record",le="10"} 2.0 58 | io_monitor_ratio_bucket{adapter="active_record",le="+Inf"} 2.0 59 | io_monitor_ratio_sum{adapter="active_record"} 0.15779381908414167 60 | io_monitor_ratio_count{adapter="active_record"} 2.0 61 | ... 62 | ``` 63 | If you want to customize Prometheus publisher you can pass it as object: 64 | ```ruby 65 | IoMonitor.configure do |config| 66 | config.publish = [ 67 | IoMonitor::PrometheusPublisher.new( 68 | registry: custom_registry, # defaults to Prometheus::Client.registry 69 | aggregation: :max, # defaults to nil 70 | buckets: [0.1, 5, 10] # defaults to Prometheus::Client::Histogram::DEFAULT_BUCKETS 71 | ) 72 | ] 73 | end 74 | ``` 75 | 76 | In addition, if `publish` is set to logs, additional data will be logged on each request: 77 | 78 | ``` 79 | Completed 200 OK in 349ms (Views: 2.1ms | ActiveRecord: 38.7ms | ActiveRecord Payload: 866.00 B | Response Payload: 25.00 B | Allocations: 72304) 80 | ``` 81 | 82 | If you want to inspect payload sizes, check out payload data for the `process_action.action_controller` event: 83 | 84 | ```ruby 85 | ActiveSupport::Notifications.subscribe("process_action.action_controller") do |name, start, finish, id, payload| 86 | payload[:io_monitor] # { active_record: 866, response: 25 } 87 | end 88 | ``` 89 | 90 | ## Per–action monitoring 91 | 92 | Since this approach can lead to false–positives or other things you don't want or cannot fix, there is a way to configure monitoring only for specific actions: 93 | 94 | ```ruby 95 | class MyController < ApplicationController 96 | include IoMonitor::Controller 97 | 98 | monitor_io_for :index, :show 99 | end 100 | ``` 101 | 102 | ## Custom publishers 103 | 104 | Implement your custom publisher by inheriting from `BasePublisher`: 105 | 106 | ```ruby 107 | class MyPublisher < IoMonitor::BasePublisher 108 | def publish(source, ratio) 109 | puts "Warn threshold reched for #{source} at #{ratio}!" 110 | end 111 | end 112 | ``` 113 | 114 | Then specify it in the configuration: 115 | 116 | ```ruby 117 | IoMonitor.configure do |config| 118 | config.publish = MyPublisher.new 119 | end 120 | ``` 121 | 122 | ## Custom adapters 123 | 124 | Implement your custom adapter by inheriting from `BaseAdapter`: 125 | 126 | ```ruby 127 | class MyAdapter < IoMonitor::BaseAdapter 128 | def self.kind 129 | :my_source 130 | end 131 | 132 | def initialize! 133 | # Take a look at `AbstractAdapterPatch` for an example. 134 | end 135 | end 136 | ``` 137 | 138 | Then specify it in the configuration: 139 | 140 | ```ruby 141 | IoMonitor.configure do |config| 142 | config.adapters = [:active_record, MyAdapter.new] 143 | end 144 | ``` 145 | 146 | ## Development 147 | 148 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. 149 | 150 | ## Credits 151 | 152 | Initially sponsored by [Evil Martians](http://evilmartians.com). 153 | 154 | ## Contributing 155 | 156 | Bug reports and pull requests are welcome on GitHub at https://github.com/DmitryTsepelev/io_monitor. 157 | 158 | ## License 159 | 160 | The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). 161 | 162 | ## Credits 163 | 164 | Thanks to [@prog-supdex](https://github.com/prog-supdex) and [@maxshend](https://github.com/maxshend) for building the initial implementations (see [PR#2](https://github.com/DmitryTsepelev/io_monitor/pull/2) and [PR#3](https://github.com/DmitryTsepelev/io_monitor/pull/3)). 165 | --------------------------------------------------------------------------------