├── .github └── workflows │ └── build.yml ├── .gitignore ├── CHANGELOG.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── gemfiles ├── activerecord71.gemfile └── activerecord72.gemfile ├── lib ├── generators │ ├── rollups_generator.rb │ └── templates │ │ ├── dimensions.rb.tt │ │ └── standard.rb.tt ├── rollup.rb ├── rollup │ ├── aggregator.rb │ ├── model.rb │ ├── utils.rb │ └── version.rb └── rollups.rb ├── rollups.gemspec └── test ├── aggregator_test.rb ├── calculation_test.rb ├── column_test.rb ├── dimensions_test.rb ├── interval_test.rb ├── query_test.rb ├── rollup_test.rb ├── rollups_generator_test.rb ├── support └── active_record.rb └── test_helper.rb /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: [push, pull_request] 3 | jobs: 4 | build: 5 | strategy: 6 | fail-fast: false 7 | matrix: 8 | include: 9 | - ruby: 3.4 10 | gemfile: Gemfile 11 | - ruby: 3.3 12 | gemfile: gemfiles/activerecord72.gemfile 13 | - ruby: 3.2 14 | gemfile: gemfiles/activerecord71.gemfile 15 | runs-on: ubuntu-latest 16 | env: 17 | BUNDLE_GEMFILE: ${{ matrix.gemfile }} 18 | steps: 19 | - uses: actions/checkout@v4 20 | - uses: ruby/setup-ruby@v1 21 | with: 22 | ruby-version: ${{ matrix.ruby }} 23 | bundler-cache: true 24 | - uses: ankane/setup-postgres@v1 25 | with: 26 | database: rollup_test 27 | - uses: ankane/setup-mysql@v1 28 | with: 29 | database: rollup_test 30 | - run: mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql 31 | - run: bundle exec rake test 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | *.lock 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.5.0 (2025-05-04) 2 | 3 | - Dropped support for Ruby < 3.2 and Active Record < 7.1 4 | 5 | ## 0.4.1 (2024-10-07) 6 | 7 | - Fixed connection leasing for Active Record 7.2+ 8 | 9 | ## 0.4.0 (2024-10-01) 10 | 11 | - Added support for Active Record 8 12 | - Dropped support for Ruby < 3.1 and Active Record < 7 13 | 14 | ## 0.3.2 (2024-02-09) 15 | 16 | - Fixed incorrect rollups with `time_zone: false` when `Rollup.time_zone` has a negative UTC offset 17 | 18 | ## 0.3.1 (2023-09-20) 19 | 20 | - Added support for Trilogy 21 | 22 | ## 0.3.0 (2023-07-02) 23 | 24 | - Dropped support for Ruby < 3 and Active Record < 6.1 25 | 26 | ## 0.2.0 (2022-04-05) 27 | 28 | - Added `range` option 29 | - Dropped support for Active Record < 5.2 30 | 31 | ## 0.1.4 (2021-12-15) 32 | 33 | - Fixed warning with Active Record 7 34 | 35 | ## 0.1.3 (2021-10-20) 36 | 37 | - Fixed issue rolling up rollups 38 | 39 | ## 0.1.2 (2021-06-07) 40 | 41 | - Fixed deprecation warning with Active Record 6.1 42 | 43 | ## 0.1.1 (2020-09-09) 44 | 45 | - Fixed results for earlier versions of MySQL 46 | 47 | ## 0.1.0 (2020-09-07) 48 | 49 | - First release 50 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec 4 | 5 | gem "rake" 6 | gem "minitest", ">= 5" 7 | gem "activerecord", "~> 8.0.0" 8 | gem "railties", require: false 9 | 10 | platform :ruby do 11 | gem "pg" 12 | gem "mysql2" 13 | gem "trilogy" 14 | gem "sqlite3" 15 | end 16 | 17 | platform :jruby do 18 | gem "sqlite3-ffi" 19 | end 20 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020-2025 Andrew Kane 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rollup 2 | 3 | :fire: Rollup time-series data in Rails 4 | 5 | Works great with [Ahoy](https://github.com/ankane/ahoy) and [Searchjoy](https://github.com/ankane/searchjoy) 6 | 7 | [![Build Status](https://github.com/ankane/rollup/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/rollup/actions) 8 | 9 | ## Installation 10 | 11 | Add this line to your application’s Gemfile: 12 | 13 | ```ruby 14 | gem "rollups" 15 | ``` 16 | 17 | And run: 18 | 19 | ```sh 20 | bundle install 21 | rails generate rollups 22 | rails db:migrate 23 | ``` 24 | 25 | ## Contents 26 | 27 | - [Getting Started](#getting-started) 28 | - [Creating Rollups](#creating-rollups) 29 | - [Querying Rollups](#querying-rollups) 30 | - [Other Topics](#other-topics) 31 | - [Examples](#examples) 32 | 33 | ## Getting Started 34 | 35 | Store the number of users created by day in the `rollups` table 36 | 37 | ```ruby 38 | User.rollup("New users") 39 | ``` 40 | 41 | Get the series 42 | 43 | ```ruby 44 | Rollup.series("New users") 45 | # { 46 | # Wed, 01 Jan 2025 => 50, 47 | # Thu, 02 Jan 2025 => 100, 48 | # Fri, 03 Jan 2025 => 34 49 | # } 50 | ``` 51 | 52 | Use a rake task or background job to create rollups on a regular basis. Don’t worry too much about naming - you can [rename](#naming) later if needed. 53 | 54 | ## Creating Rollups 55 | 56 | ### Time Column 57 | 58 | Specify the time column - `created_at` by default 59 | 60 | ```ruby 61 | User.rollup("New users", column: :joined_at) 62 | ``` 63 | 64 | Change the default column for a model 65 | 66 | ```ruby 67 | class User < ApplicationRecord 68 | self.rollup_column = :joined_at 69 | end 70 | ``` 71 | 72 | ### Time Intervals 73 | 74 | Specify the interval - `day` by default 75 | 76 | ```ruby 77 | User.rollup("New users", interval: "week") 78 | ``` 79 | 80 | And when querying 81 | 82 | ```ruby 83 | Rollup.series("New users", interval: "week") 84 | ``` 85 | 86 | Supported intervals are: 87 | 88 | - hour 89 | - day 90 | - week 91 | - month 92 | - quarter 93 | - year 94 | 95 | Or any number of minutes or seconds: 96 | 97 | - 1m, 5m, 15m 98 | - 1s, 30s, 90s 99 | 100 | Weeks start on Sunday by default. Change this with: 101 | 102 | ```ruby 103 | Rollup.week_start = :monday 104 | ``` 105 | 106 | ### Time Zones 107 | 108 | The default time zone is `Time.zone`. Change this with: 109 | 110 | ```ruby 111 | Rollup.time_zone = "Pacific Time (US & Canada)" 112 | ``` 113 | 114 | or 115 | 116 | ```ruby 117 | User.rollup("New users", time_zone: "Pacific Time (US & Canada)") 118 | ``` 119 | 120 | Time zone objects also work. To see a list of available time zones in Rails, run `rake time:zones:all`. 121 | 122 | See [date storage](#date-storage) for how dates are stored. 123 | 124 | ### Calculations 125 | 126 | Rollups use `count` by default. For other calculations, use: 127 | 128 | ```ruby 129 | Order.rollup("Revenue") { |r| r.sum(:revenue) } 130 | ``` 131 | 132 | Works with `count`, `sum`, `minimum`, `maximum`, and `average`. For `median` and `percentile`, check out [ActiveMedian](https://github.com/ankane/active_median). 133 | 134 | ### Dimensions 135 | 136 | *PostgreSQL only* 137 | 138 | Create rollups with dimensions 139 | 140 | ```ruby 141 | Order.group(:platform).rollup("Orders by platform") 142 | ``` 143 | 144 | Works with multiple groups as well 145 | 146 | ```ruby 147 | Order.group(:platform, :channel).rollup("Orders by platform and channel") 148 | ``` 149 | 150 | Dimension names are determined by the `group` clause. To set manually, use: 151 | 152 | ```ruby 153 | Order.group(:channel).rollup("Orders by source", dimension_names: ["source"]) 154 | ``` 155 | 156 | See how to [query dimensions](#multiple-series). 157 | 158 | ### Updating Data 159 | 160 | When you run a rollup for the first time, the entire series is calculated. When you run it again, newer data is added. 161 | 162 | By default, the latest interval stored for a series is recalculated, since it was likely calculated before the interval completed. Earlier intervals aren’t recalculated since the source rows may have been deleted (this also improves performance). 163 | 164 | To recalculate the last few intervals, use: 165 | 166 | ```ruby 167 | User.rollup("New users", last: 3) 168 | ``` 169 | 170 | To recalculate a time range, use: 171 | 172 | ```ruby 173 | User.rollup("New users", range: 1.week.ago.all_week) 174 | ``` 175 | 176 | To only store data for completed intervals, use: 177 | 178 | ```ruby 179 | User.rollup("New users", current: false) 180 | ``` 181 | 182 | To clear and recalculate the entire series, use: 183 | 184 | ```ruby 185 | User.rollup("New users", clear: true) 186 | ``` 187 | 188 | To delete a series, use: 189 | 190 | ```ruby 191 | Rollup.where(name: "New users", interval: "day").delete_all 192 | ``` 193 | 194 | ## Querying Rollups 195 | 196 | ### Single Series 197 | 198 | Get a series 199 | 200 | ```ruby 201 | Rollup.series("New users") 202 | ``` 203 | 204 | Specify the interval if it’s not day 205 | 206 | ```ruby 207 | Rollup.series("New users", interval: "week") 208 | ``` 209 | 210 | If a series has dimensions, they must match exactly as well 211 | 212 | ```ruby 213 | Rollup.series("Orders by platform and channel", dimensions: {platform: "Web", channel: "Search"}) 214 | ``` 215 | 216 | Get a specific time range 217 | 218 | ```ruby 219 | Rollup.where(time: Date.current.all_year).series("New Users") 220 | ``` 221 | 222 | ### Multiple Series 223 | 224 | *PostgreSQL only* 225 | 226 | Get multiple series grouped by dimensions 227 | 228 | ```ruby 229 | Rollup.multi_series("Orders by platform") 230 | ``` 231 | 232 | Specify the interval if it’s not day 233 | 234 | ```ruby 235 | Rollup.multi_series("Orders by platform", interval: "week") 236 | ``` 237 | 238 | Filter by dimensions 239 | 240 | ```ruby 241 | Rollup.where_dimensions(platform: "Web").multi_series("Orders by platform and channel") 242 | ``` 243 | 244 | Get a specific time range 245 | 246 | ```ruby 247 | Rollup.where(time: Date.current.all_year).multi_series("Orders by platform") 248 | ``` 249 | 250 | ### Raw Data 251 | 252 | Uses the `Rollup` model to query the data directly 253 | 254 | ```ruby 255 | Rollup.where(name: "New users", interval: "day") 256 | ``` 257 | 258 | ### List 259 | 260 | List names and intervals 261 | 262 | ```ruby 263 | Rollup.list 264 | ``` 265 | 266 | ### Charts 267 | 268 | Rollup works great with [Chartkick](https://github.com/ankane/chartkick) 269 | 270 | ```erb 271 | <%= line_chart Rollup.series("New users") %> 272 | ``` 273 | 274 | For multiple series, set a `name` for each series before charting 275 | 276 | ```ruby 277 | series = Rollup.multi_series("Orders by platform") 278 | series.each do |s| 279 | s[:name] = s[:dimensions]["platform"] 280 | end 281 | ``` 282 | 283 | ## Other Topics 284 | 285 | ### Naming 286 | 287 | Use any naming convention you prefer. Some ideas are: 288 | 289 | - Human - `New users` 290 | - Underscore - `new_users` 291 | - Dots - `new_users.count` 292 | 293 | Rename with: 294 | 295 | ```ruby 296 | Rollup.rename("Old name", "New name") 297 | ``` 298 | 299 | ### Date Storage 300 | 301 | Rollup stores both dates and times in the `time` column depending on the interval. For date intervals (day, week, etc), it stores `00:00:00` for the time part. Cast the `time` column to a date when querying in SQL to get the correct value. 302 | 303 | - PostgreSQL: `time::date` 304 | - MySQL: `CAST(time AS date)` 305 | - SQLite: `date(time)` 306 | 307 | ## Examples 308 | 309 | - [Ahoy](#ahoy) 310 | - [Searchjoy](#searchjoy) 311 | 312 | ### Ahoy 313 | 314 | Set the default rollup column for your models 315 | 316 | ```ruby 317 | class Ahoy::Visit < ApplicationRecord 318 | self.rollup_column = :started_at 319 | end 320 | ``` 321 | 322 | and 323 | 324 | ```ruby 325 | class Ahoy::Event < ApplicationRecord 326 | self.rollup_column = :time 327 | end 328 | ``` 329 | 330 | Hourly visits 331 | 332 | ```ruby 333 | Ahoy::Visit.rollup("Visits", interval: "hour") 334 | ``` 335 | 336 | Visits by browser 337 | 338 | ```ruby 339 | Ahoy::Visit.group(:browser).rollup("Visits by browser") 340 | ``` 341 | 342 | Unique homepage views 343 | 344 | ```ruby 345 | Ahoy::Event.where(name: "Viewed homepage").joins(:visit).rollup("Homepage views") { |r| r.distinct.count(:visitor_token) } 346 | ``` 347 | 348 | Product views 349 | 350 | ```ruby 351 | Ahoy::Event.where(name: "Viewed product").group_prop(:product_id).rollup("Product views") 352 | ``` 353 | 354 | ### Searchjoy 355 | 356 | Daily searches 357 | 358 | ```ruby 359 | Searchjoy::Search.rollup("Searches") 360 | ``` 361 | 362 | Searches by query 363 | 364 | ```ruby 365 | Searchjoy::Search.group(:normalized_query).rollup("Searches by query", dimension_names: ["query"]) 366 | ``` 367 | 368 | Conversion rate 369 | 370 | ```ruby 371 | Searchjoy::Search.rollup("Search conversion rate") { |r| r.average("(converted_at IS NOT NULL)::int") } 372 | ``` 373 | 374 | ## History 375 | 376 | View the [changelog](https://github.com/ankane/rollup/blob/master/CHANGELOG.md) 377 | 378 | ## Contributing 379 | 380 | Everyone is encouraged to help improve this project. Here are a few ways you can help: 381 | 382 | - [Report bugs](https://github.com/ankane/rollup/issues) 383 | - Fix bugs and [submit pull requests](https://github.com/ankane/rollup/pulls) 384 | - Write, clarify, or fix documentation 385 | - Suggest or add new features 386 | 387 | To get started with development: 388 | 389 | ```sh 390 | git clone https://github.com/ankane/rollup.git 391 | cd rollup 392 | bundle install 393 | 394 | # create databases 395 | createdb rollup_test 396 | mysqladmin create rollup_test 397 | 398 | # run tests 399 | bundle exec rake test 400 | ``` 401 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rake/testtask" 3 | 4 | ADAPTERS = %w(postgresql mysql trilogy sqlite) 5 | 6 | ADAPTERS.each do |adapter| 7 | namespace :test do 8 | task("env:#{adapter}") { ENV["ADAPTER"] = adapter } 9 | 10 | Rake::TestTask.new(adapter => "env:#{adapter}") do |t| 11 | t.description = "Run tests for #{adapter}" 12 | t.libs << "test" 13 | t.test_files = FileList["test/**/*_test.rb"] 14 | end 15 | end 16 | end 17 | 18 | desc "Run all adapter tests" 19 | task :test do 20 | ADAPTERS.each do |adapter| 21 | next if RUBY_ENGINE == "jruby" && adapter != "sqlite" 22 | Rake::Task["test:#{adapter}"].invoke 23 | end 24 | end 25 | 26 | task default: :test 27 | -------------------------------------------------------------------------------- /gemfiles/activerecord71.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec path: ".." 4 | 5 | gem "rake" 6 | gem "minitest", ">= 5" 7 | gem "activerecord", "~> 7.1.0" 8 | gem "pg" 9 | gem "mysql2" 10 | gem "trilogy" 11 | gem "sqlite3", "< 2" 12 | gem "railties", require: false 13 | -------------------------------------------------------------------------------- /gemfiles/activerecord72.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec path: ".." 4 | 5 | gem "rake" 6 | gem "minitest", ">= 5" 7 | gem "activerecord", "~> 7.2.0" 8 | gem "pg" 9 | gem "mysql2" 10 | gem "trilogy" 11 | gem "sqlite3" 12 | gem "railties", require: false 13 | -------------------------------------------------------------------------------- /lib/generators/rollups_generator.rb: -------------------------------------------------------------------------------- 1 | require "rails/generators/active_record" 2 | 3 | # use rollups instead of rollup:install to avoid 4 | # class Rollup < ActiveRecord::Base 5 | # also works out nicely since it's the gem name 6 | class RollupsGenerator < Rails::Generators::Base 7 | include ActiveRecord::Generators::Migration 8 | source_root File.join(__dir__, "templates") 9 | 10 | def copy_templates 11 | migration_template migration_source, "db/migrate/create_rollups.rb", migration_version: migration_version 12 | end 13 | 14 | def migration_source 15 | case adapter 16 | when /postg/i 17 | "dimensions.rb" 18 | else 19 | "standard.rb" 20 | end 21 | end 22 | 23 | def migration_version 24 | "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]" 25 | end 26 | 27 | def adapter 28 | ActiveRecord::Base.connection_db_config.adapter.to_s 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/generators/templates/dimensions.rb.tt: -------------------------------------------------------------------------------- 1 | class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %> 2 | def change 3 | create_table :rollups do |t| 4 | t.string :name, null: false 5 | t.string :interval, null: false 6 | t.datetime :time, null: false 7 | t.jsonb :dimensions, null: false, default: {} 8 | t.float :value 9 | end 10 | add_index :rollups, [:name, :interval, :time, :dimensions], unique: true 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/generators/templates/standard.rb.tt: -------------------------------------------------------------------------------- 1 | class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %> 2 | def change 3 | create_table :rollups do |t| 4 | t.string :name, null: false 5 | t.string :interval, null: false 6 | t.datetime :time, null: false 7 | t.float :value 8 | end 9 | add_index :rollups, [:name, :interval, :time], unique: true 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/rollup.rb: -------------------------------------------------------------------------------- 1 | class Rollup < ActiveRecord::Base 2 | validates :name, presence: true 3 | validates :interval, presence: true 4 | validates :time, presence: true 5 | 6 | class << self 7 | attr_accessor :week_start 8 | attr_writer :time_zone 9 | end 10 | self.week_start = :sunday 11 | 12 | class << self 13 | # do not memoize so Time.zone can change 14 | def time_zone 15 | (defined?(@time_zone) && @time_zone) || Time.zone || "Etc/UTC" 16 | end 17 | 18 | def series(name, interval: "day", dimensions: {}) 19 | Utils.check_dimensions if dimensions.any? 20 | 21 | relation = where(name: name, interval: interval) 22 | relation = relation.where(dimensions: dimensions) if Utils.dimensions_supported? 23 | 24 | # use select_all due to incorrect casting with pluck 25 | sql = relation.order(:time).select(Utils.time_sql(interval), :value).to_sql 26 | result = connection_pool.with_connection { |c| c.select_all(sql) }.rows 27 | 28 | Utils.make_series(result, interval) 29 | end 30 | 31 | def multi_series(name, interval: "day") 32 | Utils.check_dimensions 33 | 34 | relation = where(name: name, interval: interval) 35 | 36 | # use select_all to reduce allocations 37 | sql = relation.order(:time).select(Utils.time_sql(interval), :value, :dimensions).to_sql 38 | result = connection_pool.with_connection { |c| c.select_all(sql) }.rows 39 | 40 | result.group_by { |r| JSON.parse(r[2]) }.map do |dimensions, rollups| 41 | {dimensions: dimensions, data: Utils.make_series(rollups, interval)} 42 | end 43 | end 44 | 45 | def where_dimensions(dimensions) 46 | Utils.check_dimensions 47 | 48 | relation = self 49 | dimensions.each do |k, v| 50 | k = k.to_s 51 | relation = 52 | if v.nil? 53 | relation.where("dimensions ->> ? IS NULL", k) 54 | elsif v.is_a?(Array) 55 | relation.where("dimensions ->> ? IN (?)", k, v.map { |vi| vi.as_json.to_s }) 56 | else 57 | relation.where("dimensions ->> ? = ?", k, v.as_json.to_s) 58 | end 59 | end 60 | relation 61 | end 62 | 63 | def list 64 | select(:name, :interval).distinct.order(:name, :interval).map do |r| 65 | {name: r.name, interval: r.interval} 66 | end 67 | end 68 | 69 | # TODO maybe use in_batches 70 | def rename(old_name, new_name) 71 | where(name: old_name).update_all(name: new_name) 72 | end 73 | end 74 | 75 | # feels cleaner than overriding _read_attribute 76 | def inspect 77 | if Utils.date_interval?(interval) 78 | super.sub(/time: "[^"]+"/, "time: \"#{time.to_formatted_s(:db)}\"") 79 | else 80 | super 81 | end 82 | end 83 | 84 | def time 85 | if Utils.date_interval?(interval) && !time_before_type_cast.nil? 86 | if time_before_type_cast.is_a?(Time) 87 | time_before_type_cast.utc.to_date 88 | else 89 | Date.parse(time_before_type_cast.to_s) 90 | end 91 | else 92 | super 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /lib/rollup/aggregator.rb: -------------------------------------------------------------------------------- 1 | class Rollup 2 | class Aggregator 3 | def initialize(klass) 4 | @klass = klass # or relation 5 | end 6 | 7 | def rollup(name, column: nil, interval: "day", dimension_names: nil, time_zone: nil, current: nil, last: nil, clear: false, range: nil, &block) 8 | raise "Name can't be blank" if name.blank? 9 | 10 | column ||= @klass.rollup_column || :created_at 11 | # Groupdate 6+ validates, but keep this for now for additional safety 12 | # no need to quote/resolve column here, as Groupdate handles it 13 | column = validate_column(column) 14 | 15 | relation = perform_group(name, column: column, interval: interval, time_zone: time_zone, current: current, last: last, clear: clear, range: range) 16 | result = perform_calculation(relation, &block) 17 | 18 | dimension_names = set_dimension_names(dimension_names, relation) 19 | records = prepare_result(result, name, dimension_names, interval) 20 | 21 | maybe_clear(clear, name, interval) do 22 | save_records(records) if records.any? 23 | end 24 | end 25 | 26 | # basic version of Active Record disallow_raw_sql! 27 | # symbol = column (safe), Arel node = SQL (safe), other = untrusted 28 | # matches table.column and column 29 | def validate_column(column) 30 | unless column.is_a?(Symbol) || column.is_a?(Arel::Nodes::SqlLiteral) 31 | column = column.to_s 32 | unless /\A\w+(\.\w+)?\z/i.match?(column) 33 | raise ActiveRecord::UnknownAttributeReference, "Query method called with non-attribute argument(s): #{column.inspect}. Use Arel.sql() for known-safe values." 34 | end 35 | end 36 | column 37 | end 38 | 39 | def perform_group(name, column:, interval:, time_zone:, current:, last:, clear:, range:) 40 | raise ArgumentError, "Cannot use last and range together" if last && range 41 | raise ArgumentError, "Cannot use last and clear together" if last && clear 42 | raise ArgumentError, "Cannot use range and clear together" if range && clear 43 | raise ArgumentError, "Cannot use range and current together" if range && !current.nil? 44 | 45 | current = true if current.nil? 46 | time_zone = Rollup.time_zone if time_zone.nil? 47 | 48 | gd_options = { 49 | current: current 50 | } 51 | 52 | # make sure Groupdate global options aren't applied 53 | gd_options[:time_zone] = time_zone 54 | gd_options[:week_start] = Rollup.week_start if interval.to_s == "week" 55 | gd_options[:day_start] = 0 if Utils.date_interval?(interval) 56 | 57 | if last 58 | gd_options[:last] = last 59 | elsif range 60 | gd_options[:range] = range 61 | gd_options[:expand_range] = true 62 | gd_options.delete(:current) 63 | elsif !clear 64 | # if no rollups, compute all intervals 65 | # if rollups, recompute last interval 66 | max_time = Rollup.unscoped.where(name: name, interval: interval).maximum(Utils.time_sql(interval)) 67 | if max_time 68 | # for MySQL on Ubuntu 18.04 (and likely other platforms) 69 | if max_time.is_a?(String) 70 | utc = ActiveSupport::TimeZone["Etc/UTC"] 71 | max_time = 72 | if Utils.date_interval?(interval) 73 | max_time.to_date 74 | else 75 | t = utc.parse(max_time) 76 | t = t.in_time_zone(time_zone) if time_zone 77 | t 78 | end 79 | end 80 | 81 | # aligns perfectly if time zone doesn't change 82 | # if time zone does change, there are other problems besides this 83 | gd_options[:range] = max_time.. 84 | end 85 | end 86 | 87 | # intervals are stored as given 88 | # we don't normalize intervals (i.e. change 60s -> 1m) 89 | case interval.to_s 90 | when "hour", "day", "week", "month", "quarter", "year" 91 | @klass.group_by_period(interval, column, **gd_options) 92 | when /\A\d+s\z/ 93 | @klass.group_by_second(column, n: interval.to_i, **gd_options) 94 | when /\A\d+m\z/ 95 | @klass.group_by_minute(column, n: interval.to_i, **gd_options) 96 | else 97 | raise ArgumentError, "Invalid interval: #{interval}" 98 | end 99 | end 100 | 101 | def set_dimension_names(dimension_names, relation) 102 | groups = relation.group_values[0..-2] 103 | 104 | if dimension_names 105 | Utils.check_dimensions 106 | if dimension_names.size != groups.size 107 | raise ArgumentError, "Expected dimension_names to be size #{groups.size}, not #{dimension_names.size}" 108 | end 109 | dimension_names 110 | else 111 | Utils.check_dimensions if groups.any? 112 | groups.map { |group| determine_dimension_name(group) } 113 | end 114 | end 115 | 116 | def determine_dimension_name(group) 117 | # split by ., ->>, and -> and remove whitespace 118 | value = group.to_s.split(/\s*((\.)|(->>)|(->))\s*/).last 119 | 120 | # removing starting and ending quotes 121 | # for simplicity, they don't need to be the same 122 | value = value[1..-2] if value.match?(/\A["'`].+["'`]\z/) 123 | 124 | unless value.match?(/\A\w+\z/) 125 | raise "Cannot determine dimension name: #{group}. Use the dimension_names option" 126 | end 127 | 128 | value 129 | end 130 | 131 | # calculation can mutate relation, but that's fine 132 | def perform_calculation(relation, &block) 133 | if block_given? 134 | yield(relation) 135 | else 136 | relation.count 137 | end 138 | end 139 | 140 | def prepare_result(result, name, dimension_names, interval) 141 | raise "Expected calculation to return Hash, not #{result.class.name}" unless result.is_a?(Hash) 142 | 143 | time_class = Utils.date_interval?(interval) ? Date : Time 144 | dimensions_supported = Utils.dimensions_supported? 145 | expected_key_size = dimension_names.size + 1 146 | 147 | result.map do |key, value| 148 | dimensions = {} 149 | if dimensions_supported && dimension_names.any? 150 | unless key.is_a?(Array) && key.size == expected_key_size 151 | raise "Expected result key to be Array with size #{expected_key_size}" 152 | end 153 | time = key[-1] 154 | # may be able to support dimensions in SQLite by sorting dimension names 155 | dimension_names.each_with_index do |dn, i| 156 | dimensions[dn] = key[i] 157 | end 158 | else 159 | time = key 160 | end 161 | 162 | raise "Expected time to be #{time_class.name}, not #{time.class.name}" unless time.is_a?(time_class) 163 | raise "Expected value to be Numeric or nil, not #{value.class.name}" unless value.is_a?(Numeric) || value.nil? 164 | 165 | record = { 166 | name: name, 167 | interval: interval, 168 | time: time, 169 | value: value 170 | } 171 | record[:dimensions] = dimensions if dimensions_supported 172 | record 173 | end 174 | end 175 | 176 | def maybe_clear(clear, name, interval) 177 | if clear 178 | Rollup.transaction do 179 | Rollup.unscoped.where(name: name, interval: interval).delete_all 180 | yield 181 | end 182 | else 183 | yield 184 | end 185 | end 186 | 187 | def save_records(records) 188 | # order must match unique index 189 | # consider using index name instead 190 | conflict_target = [:name, :interval, :time] 191 | conflict_target << :dimensions if Utils.dimensions_supported? 192 | 193 | options = Utils.mysql? ? {} : {unique_by: conflict_target} 194 | if ActiveRecord::VERSION::MAJOR >= 8 195 | utc = ActiveSupport::TimeZone["Etc/UTC"] 196 | records.each do |v| 197 | v[:time] = v[:time].in_time_zone(utc) if v[:time].is_a?(Date) 198 | end 199 | end 200 | Rollup.unscoped.upsert_all(records, **options) 201 | end 202 | end 203 | end 204 | -------------------------------------------------------------------------------- /lib/rollup/model.rb: -------------------------------------------------------------------------------- 1 | class Rollup 2 | module Model 3 | attr_accessor :rollup_column 4 | 5 | def rollup(*args, **options, &block) 6 | Aggregator.new(self).rollup(*args, **options, &block) 7 | nil 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/rollup/utils.rb: -------------------------------------------------------------------------------- 1 | class Rollup 2 | module Utils 3 | DATE_INTERVALS = %w(day week month quarter year) 4 | 5 | class << self 6 | def time_sql(interval) 7 | if date_interval?(interval) 8 | if postgresql? 9 | "rollups.time::date" 10 | elsif sqlite? 11 | "date(rollups.time)" 12 | else 13 | "CAST(rollups.time AS date)" 14 | end 15 | else 16 | :time 17 | end 18 | end 19 | 20 | def date_interval?(interval) 21 | DATE_INTERVALS.include?(interval.to_s) 22 | end 23 | 24 | def dimensions_supported? 25 | unless defined?(@dimensions_supported) 26 | @dimensions_supported = postgresql? && Rollup.column_names.include?("dimensions") 27 | end 28 | @dimensions_supported 29 | end 30 | 31 | def check_dimensions 32 | raise "Dimensions not supported" unless dimensions_supported? 33 | end 34 | 35 | def adapter_name 36 | Rollup.connection_db_config.adapter.to_s 37 | end 38 | 39 | def postgresql? 40 | adapter_name =~ /postg/i 41 | end 42 | 43 | def mysql? 44 | adapter_name =~ /mysql|trilogy/i 45 | end 46 | 47 | def sqlite? 48 | adapter_name =~ /sqlite/i 49 | end 50 | 51 | def make_series(result, interval) 52 | series = {} 53 | if Utils.date_interval?(interval) 54 | result.each do |row| 55 | series[row[0].to_date] = row[1] 56 | end 57 | else 58 | time_zone = Rollup.time_zone 59 | if result.any? && result[0][0].is_a?(Time) 60 | result.each do |row| 61 | series[row[0].in_time_zone(time_zone)] = row[1] 62 | end 63 | else 64 | utc = ActiveSupport::TimeZone["Etc/UTC"] 65 | result.each do |row| 66 | # row can be time or string 67 | series[utc.parse(row[0]).in_time_zone(time_zone)] = row[1] 68 | end 69 | end 70 | end 71 | series 72 | end 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/rollup/version.rb: -------------------------------------------------------------------------------- 1 | class Rollup 2 | # not used in gemspec to avoid superclass mismatch 3 | # be sure to update there as well 4 | VERSION = "0.5.0" 5 | end 6 | -------------------------------------------------------------------------------- /lib/rollups.rb: -------------------------------------------------------------------------------- 1 | # dependencies 2 | require "active_support" 3 | require "groupdate" 4 | 5 | ActiveSupport.on_load(:active_record) do 6 | # must come first 7 | require_relative "rollup" 8 | 9 | require_relative "rollup/model" 10 | extend Rollup::Model 11 | Rollup.rollup_column = :time 12 | 13 | # modules 14 | require_relative "rollup/aggregator" 15 | require_relative "rollup/utils" 16 | require_relative "rollup/version" 17 | end 18 | -------------------------------------------------------------------------------- /rollups.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |spec| 2 | spec.name = "rollups" 3 | spec.version = "0.5.0" 4 | spec.summary = "Rollup time-series data in Rails" 5 | spec.homepage = "https://github.com/ankane/rollup" 6 | spec.license = "MIT" 7 | 8 | spec.author = "Andrew Kane" 9 | spec.email = "andrew@ankane.org" 10 | 11 | spec.files = Dir["*.{md,txt}", "{lib}/**/*"] 12 | spec.require_path = "lib" 13 | 14 | spec.required_ruby_version = ">= 3.2" 15 | 16 | spec.add_dependency "activesupport", ">= 7.1" 17 | spec.add_dependency "groupdate", ">= 6.1" 18 | end 19 | -------------------------------------------------------------------------------- /test/aggregator_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class AggregatorTest < Minitest::Test 4 | def test_updates_latest 5 | create_users 6 | User.rollup("Test") 7 | 8 | create_users 9 | User.rollup("Test") 10 | 11 | expected = { 12 | now.to_date - 2 => 1, 13 | now.to_date - 1 => 1, 14 | now.to_date => 2 15 | } 16 | assert_equal expected, Rollup.series("Test") 17 | end 18 | 19 | def test_clear 20 | create_users 21 | User.rollup("Test") 22 | 23 | create_users 24 | User.rollup("Test", clear: true) 25 | 26 | expected = { 27 | now.to_date - 2 => 2, 28 | now.to_date - 1 => 2, 29 | now.to_date => 2 30 | } 31 | assert_equal expected, Rollup.series("Test") 32 | end 33 | 34 | def test_last 35 | create_users 36 | User.rollup("Test") 37 | 38 | create_users 39 | User.rollup("Test", last: 2) 40 | 41 | expected = { 42 | now.to_date - 2 => 1, 43 | now.to_date - 1 => 2, 44 | now.to_date => 2 45 | } 46 | assert_equal expected, Rollup.series("Test") 47 | end 48 | 49 | def test_last_clear 50 | error = assert_raises(ArgumentError) do 51 | User.rollup("Test", last: 2, clear: true) 52 | end 53 | assert_equal "Cannot use last and clear together", error.message 54 | end 55 | 56 | def test_last_range 57 | error = assert_raises(ArgumentError) do 58 | User.rollup("Test", last: 2, range: now.all_day) 59 | end 60 | assert_equal "Cannot use last and range together", error.message 61 | end 62 | 63 | def test_current 64 | create_users 65 | User.rollup("Test", current: false) 66 | 67 | expected = { 68 | now.to_date - 2 => 1, 69 | now.to_date - 1 => 1 70 | } 71 | assert_equal expected, Rollup.series("Test") 72 | end 73 | 74 | def test_name_nil 75 | error = assert_raises do 76 | User.rollup(nil) 77 | end 78 | assert_equal error.message, "Name can't be blank" 79 | end 80 | 81 | def test_name_empty 82 | error = assert_raises do 83 | User.rollup("") 84 | end 85 | assert_equal error.message, "Name can't be blank" 86 | end 87 | 88 | def test_range 89 | create_users 90 | User.rollup("Test", range: (now - 1.day).all_day) 91 | 92 | expected = { 93 | now.to_date - 1 => 1 94 | } 95 | assert_equal expected, Rollup.series("Test") 96 | end 97 | 98 | def test_range_updates 99 | create_users 100 | User.rollup("Test") 101 | 102 | create_users 103 | User.rollup("Test", range: (now - 1.day).all_day) 104 | 105 | expected = { 106 | now.to_date - 2 => 1, 107 | now.to_date - 1 => 2, 108 | now.to_date => 1 109 | } 110 | assert_equal expected, Rollup.series("Test") 111 | end 112 | 113 | def test_range_expanded 114 | create_users 115 | User.rollup("Test", range: now...now) 116 | 117 | expected = { 118 | now.to_date => 1 119 | } 120 | assert_equal expected, Rollup.series("Test") 121 | end 122 | 123 | def test_range_clear 124 | error = assert_raises(ArgumentError) do 125 | User.rollup("Test", range: now.all_day, clear: true) 126 | end 127 | assert_equal "Cannot use range and clear together", error.message 128 | end 129 | 130 | def test_range_current 131 | error = assert_raises(ArgumentError) do 132 | User.rollup("Test", range: now.all_day, current: false) 133 | end 134 | assert_equal "Cannot use range and current together", error.message 135 | end 136 | end 137 | -------------------------------------------------------------------------------- /test/calculation_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class CalculationTest < Minitest::Test 4 | def test_count 5 | User.create!(created_at: now - 2.days) 6 | User.create!(created_at: now) 7 | User.create!(created_at: now) 8 | User.rollup("Test") 9 | expected = { 10 | now.to_date - 2 => 1, 11 | now.to_date - 1 => 0, 12 | now.to_date => 2 13 | } 14 | assert_equal expected, Rollup.series("Test") 15 | end 16 | 17 | def test_sum 18 | User.create!(created_at: now - 2.days, visits: 1) 19 | User.create!(created_at: now, visits: 2) 20 | User.create!(created_at: now, visits: 3) 21 | User.rollup("Test") { |r| r.sum(:visits) } 22 | expected = { 23 | now.to_date - 2 => 1, 24 | now.to_date - 1 => 0, 25 | now.to_date => 5 26 | } 27 | assert_equal expected, Rollup.series("Test") 28 | end 29 | 30 | def test_average 31 | User.create!(created_at: now - 2.days, visits: 1) 32 | User.create!(created_at: now, visits: 2) 33 | User.create!(created_at: now, visits: 3) 34 | User.rollup("Test") { |r| r.average(:visits) } 35 | expected = { 36 | now.to_date - 2 => 1, 37 | now.to_date - 1 => nil, 38 | now.to_date => 2.5 39 | } 40 | assert_equal expected, Rollup.series("Test") 41 | end 42 | 43 | def test_minimum 44 | User.create!(created_at: now - 2.days, visits: 1) 45 | User.create!(created_at: now, visits: 2) 46 | User.create!(created_at: now, visits: 3) 47 | User.rollup("Test") { |r| r.minimum(:visits) } 48 | expected = { 49 | now.to_date - 2 => 1, 50 | now.to_date - 1 => nil, 51 | now.to_date => 2 52 | } 53 | assert_equal expected, Rollup.series("Test") 54 | end 55 | 56 | def test_maximum 57 | User.create!(created_at: now - 2.days, visits: 1) 58 | User.create!(created_at: now, visits: 2) 59 | User.create!(created_at: now, visits: 3) 60 | User.rollup("Test") { |r| r.maximum(:visits) } 61 | expected = { 62 | now.to_date - 2 => 1, 63 | now.to_date - 1 => nil, 64 | now.to_date => 3 65 | } 66 | assert_equal expected, Rollup.series("Test") 67 | end 68 | 69 | def test_bad_type 70 | error = assert_raises do 71 | User.rollup("Test") { Object.new } 72 | end 73 | assert_equal "Expected calculation to return Hash, not Object", error.message 74 | end 75 | 76 | def test_bad_key_date 77 | error = assert_raises do 78 | User.rollup("Test") { {"non-date" => 1} } 79 | end 80 | assert_equal "Expected time to be Date, not String", error.message 81 | end 82 | 83 | def test_bad_key_time 84 | error = assert_raises do 85 | User.rollup("Test", interval: "hour") { {"non-time" => 1} } 86 | end 87 | assert_equal "Expected time to be Time, not String", error.message 88 | end 89 | 90 | def test_bad_key_type 91 | skip unless dimensions_supported? 92 | 93 | error = assert_raises do 94 | User.group(:browser).rollup("Test") { {Date.current => 1} } 95 | end 96 | assert_equal "Expected result key to be Array with size 2", error.message 97 | end 98 | 99 | def test_bad_key_size 100 | skip unless dimensions_supported? 101 | 102 | error = assert_raises do 103 | User.group(:browser).rollup("Test") { {[Date.current] => 1} } 104 | end 105 | assert_equal "Expected result key to be Array with size 2", error.message 106 | end 107 | 108 | def test_bad_value 109 | error = assert_raises do 110 | User.rollup("Test") { {Date.current => "string"} } 111 | end 112 | assert_equal "Expected value to be Numeric or nil, not String", error.message 113 | end 114 | end 115 | -------------------------------------------------------------------------------- /test/column_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class ColumnTest < Minitest::Test 4 | def test_string 5 | User.rollup("Test", column: "created_at") 6 | end 7 | 8 | def test_string_table 9 | User.rollup("Test", column: "users.created_at") 10 | end 11 | 12 | def test_symbol 13 | User.rollup("Test", column: :created_at) 14 | end 15 | 16 | def test_symbol_missing 17 | # for sqlite, double-quoted string literals are accepted 18 | # https://www.sqlite.org/quirks.html 19 | if sqlite? 20 | User.rollup("Test", column: :missing) 21 | assert_equal 0, Rollup.count 22 | else 23 | assert_raises do 24 | User.rollup("Test", column: :missing) 25 | end 26 | end 27 | end 28 | 29 | def test_symbol_quoted 30 | User.rollup("Test", column: :missing) rescue nil 31 | sql = $sql.last 32 | quoted_name = User.connection.quote_column_name("missing") 33 | refute_equal quoted_name, "missing" 34 | assert_match quoted_name, sql 35 | # important: makes sure all instances are quoted 36 | assert_equal sql.split(/[^_]missing/).size, sql.split(quoted_name).size 37 | end 38 | 39 | def test_arel 40 | User.rollup("Test", column: Arel.sql("(created_at)")) 41 | end 42 | 43 | def test_no_arel 44 | error = assert_raises(ActiveRecord::UnknownAttributeReference) do 45 | User.rollup("Test", column: "(created_at)") 46 | end 47 | assert_equal "Query method called with non-attribute argument(s): \"(created_at)\". Use Arel.sql() for known-safe values.", error.message 48 | end 49 | 50 | def test_rollup_column 51 | Post.create! 52 | Post.rollup("Test") 53 | end 54 | 55 | def test_joins 56 | user = User.create!(created_at: now - 2.days) 57 | user.posts.create! 58 | user.posts.create! 59 | 60 | user2 = User.create!(created_at: now - 1.days) 61 | user2.posts.create! 62 | 63 | User.create!(created_at: now) 64 | Post.create! 65 | 66 | Post.joins(:user).rollup("Test", column: "users.created_at") 67 | 68 | expected = { 69 | now.to_date - 2 => 2, 70 | now.to_date - 1 => 1 71 | } 72 | assert_equal expected, Rollup.series("Test") 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /test/dimensions_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class DimensionsTest < Minitest::Test 4 | def setup 5 | skip unless dimensions_supported? 6 | super 7 | end 8 | 9 | def test_string 10 | assert_dimension_name "browser", "browser" 11 | end 12 | 13 | def test_string_table 14 | assert_dimension_name "users.browser", "browser" 15 | end 16 | 17 | def test_string_quoted 18 | assert_dimension_name "\"browser\"", "browser" 19 | end 20 | 21 | def test_string_table_quoted 22 | assert_dimension_name "\"users\".\"browser\"", "browser" 23 | end 24 | 25 | def test_symbol 26 | assert_dimension_name :browser, "browser" 27 | end 28 | 29 | def test_json 30 | assert_dimension_name "properties -> 'browser'", "browser" 31 | end 32 | 33 | def test_json_text_operator 34 | assert_dimension_name "properties ->> 'browser'", "browser" 35 | end 36 | 37 | def test_unknown 38 | error = assert_raises do 39 | User.group("LOWER(browser)").rollup("Test") 40 | end 41 | assert_equal "Cannot determine dimension name: LOWER(browser). Use the dimension_names option", error.message 42 | end 43 | 44 | def test_dimension_names 45 | assert_dimension_name :browser, "hi", dimension_names: ["hi"] 46 | end 47 | 48 | def test_dimension_names_wrong_size 49 | error = assert_raises(ArgumentError) do 50 | User.group(:browser).rollup("Test", dimension_names: ["a", "b"]) 51 | end 52 | assert_equal "Expected dimension_names to be size 1, not 2", error.message 53 | end 54 | 55 | def assert_dimension_name(group, expected, **options) 56 | User.create!(browser: "Firefox", properties: {browser: "Firefox"}) 57 | User.group(group).rollup("Test", **options) 58 | assert_equal [expected], Rollup.last.dimensions.keys 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /test/interval_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class IntervalTest < Minitest::Test 4 | def test_1s 5 | assert_interval "1s" 6 | end 7 | 8 | def test_30s 9 | assert_interval "30s" 10 | end 11 | 12 | def test_1m 13 | assert_interval "1m" 14 | end 15 | 16 | def test_5m 17 | assert_interval "5m" 18 | end 19 | 20 | def test_hour 21 | assert_interval "hour" 22 | end 23 | 24 | def test_day 25 | assert_interval "day" 26 | end 27 | 28 | def test_week 29 | assert_interval "week" 30 | end 31 | 32 | def test_month 33 | assert_interval "month" 34 | end 35 | 36 | def test_quarter 37 | skip "Not supported by Groupdate" if sqlite? 38 | 39 | assert_interval "quarter" 40 | end 41 | 42 | def test_year 43 | assert_interval "year" 44 | end 45 | 46 | def test_bad 47 | error = assert_raises(ArgumentError) do 48 | User.rollup("Test", interval: "bad") 49 | end 50 | assert_equal "Invalid interval: bad", error.message 51 | end 52 | 53 | def assert_interval(interval) 54 | create_users(interval: interval) 55 | User.rollup("Test", interval: interval) 56 | 57 | assert [interval]*3, Rollup.pluck(:interval) 58 | 59 | start = 60 | case interval 61 | when "1s" 62 | now.change(nsec: 0) 63 | when "30s" 64 | Time.at(now.to_i / 30 * 30) 65 | when "1m" 66 | now.change(sec: 0) 67 | when "5m" 68 | Time.at(now.to_i / 5.minutes * 5.minutes) 69 | when "week" 70 | now.beginning_of_week(:sunday) 71 | else 72 | now.send("beginning_of_#{interval}") 73 | end 74 | 75 | step = interval_step(interval) 76 | 77 | expected = {} 78 | 3.times do |i| 79 | k = start.dup 80 | # need to go one step at a time 81 | i.times do 82 | k -= step 83 | end 84 | k = k.to_date if %w(day week month quarter year).include?(interval) 85 | expected[k] = 1 86 | end 87 | assert_equal expected, Rollup.series("Test", interval: interval) 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /test/query_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class QueryTest < Minitest::Test 4 | def test_series 5 | create_users 6 | User.rollup("Test") 7 | expected = {} 8 | 3.times do |i| 9 | expected[now.to_date - i] = 1 10 | end 11 | assert_equal expected, Rollup.series("Test") 12 | assert_empty Rollup.series("Test", interval: "week") 13 | end 14 | 15 | def test_series_where 16 | create_users 17 | User.rollup("Test") 18 | expected = {now.to_date - 1 => 1} 19 | range = (now.to_date - 1)...now.to_date 20 | assert_equal expected, Rollup.where(time: range).series("Test") 21 | end 22 | 23 | def test_series_missing 24 | assert_empty Rollup.series("Test") 25 | end 26 | 27 | def test_multi_series_no_dimensions 28 | skip unless dimensions_supported? 29 | 30 | create_users 31 | User.rollup("Test") 32 | data = {} 33 | 3.times do |i| 34 | data[now.to_date - i] = 1 35 | end 36 | expected = [{dimensions: {}, data: data}] 37 | assert_equal expected, Rollup.multi_series("Test") 38 | end 39 | 40 | def test_multi_series_one_dimension 41 | skip unless dimensions_supported? 42 | 43 | browsers = %w(Firefox Firefox Brave) 44 | create_users(browser: browsers) 45 | User.group(:browser).rollup("Test") 46 | 47 | brave_data = {} 48 | firefox_data = {} 49 | 3.times do |i| 50 | brave_data[now.to_date - i] = browsers[i] == "Brave" ? 1 : 0 51 | firefox_data[now.to_date - i] = browsers[i] == "Firefox" ? 1 : 0 52 | end 53 | expected = [ 54 | {dimensions: {"browser" => "Brave"}, data: brave_data}, 55 | {dimensions: {"browser" => "Firefox"}, data: firefox_data} 56 | ] 57 | assert_equal expected, Rollup.multi_series("Test").sort_by { |s| s[:dimensions]["browser"] } 58 | end 59 | 60 | def test_multi_series_many_dimensions 61 | skip unless dimensions_supported? 62 | 63 | browsers = %w(Firefox Firefox Brave) 64 | oses = %w(Mac Linux Linux) 65 | create_users(browser: browsers, os: oses) 66 | User.group(:browser, :os).rollup("Test") 67 | 68 | brave_linux_data = {} 69 | firefox_linux_data = {} 70 | firefox_mac_data = {} 71 | 3.times do |i| 72 | brave_linux_data[now.to_date - i] = browsers[i] == "Brave" && oses[i] == "Linux" ? 1 : 0 73 | firefox_linux_data[now.to_date - i] = browsers[i] == "Firefox" && oses[i] == "Linux" ? 1 : 0 74 | firefox_mac_data[now.to_date - i] = browsers[i] == "Firefox" && oses[i] == "Mac" ? 1 : 0 75 | end 76 | expected = [ 77 | {dimensions: {"browser" => "Brave", "os" => "Linux"}, data: brave_linux_data}, 78 | {dimensions: {"browser" => "Firefox", "os" => "Linux"}, data: firefox_linux_data}, 79 | {dimensions: {"browser" => "Firefox", "os" => "Mac"}, data: firefox_mac_data} 80 | ] 81 | assert_equal expected, Rollup.multi_series("Test").sort_by { |s| [s[:dimensions]["browser"], s[:dimensions]["os"]] } 82 | end 83 | 84 | def test_multi_series_dimensions_numeric 85 | skip unless dimensions_supported? 86 | 87 | visits = [3, 3, 5] 88 | create_users(visits: visits) 89 | User.group(:visits).rollup("Test") 90 | 91 | three_data = {} 92 | five_data = {} 93 | 3.times do |i| 94 | three_data[now.to_date - i] = visits[i] == 3 ? 1 : 0 95 | five_data[now.to_date - i] = visits[i] == 5 ? 1 : 0 96 | end 97 | expected = [ 98 | {dimensions: {"visits" => 3}, data: three_data}, 99 | {dimensions: {"visits" => 5}, data: five_data} 100 | ] 101 | assert_equal expected, Rollup.multi_series("Test").sort_by { |s| s[:dimensions]["visits"] } 102 | end 103 | 104 | def test_multi_series_missing 105 | skip unless dimensions_supported? 106 | 107 | assert_empty Rollup.multi_series("Test") 108 | end 109 | 110 | def test_where_dimensions 111 | skip unless dimensions_supported? 112 | 113 | browsers = %w(Firefox Firefox Brave) 114 | oses = %w(Mac Linux Linux) 115 | visits = [3, 3, 5] 116 | create_users(browser: browsers, os: oses, visits: visits) 117 | User.group(:browser, :os, :visits).rollup("Test") 118 | 119 | assert_equal 2, Rollup.where_dimensions(browser: "Firefox").sum(:value) 120 | assert_equal 3, Rollup.where_dimensions(browser: ["Firefox", "Brave"]).sum(:value) 121 | assert_equal 1, Rollup.where_dimensions(browser: "Brave").sum(:value) 122 | assert_equal 1, Rollup.where_dimensions(browser: "Brave", os: "Linux").sum(:value) 123 | assert_equal 0, Rollup.where_dimensions(browser: "Brave", os: "Mac").sum(:value) 124 | assert_equal 2, Rollup.where_dimensions(visits: 3).sum(:value) 125 | assert_equal 3, Rollup.where_dimensions(visits: [3, 5]).sum(:value) 126 | end 127 | 128 | def test_list 129 | create_users 130 | User.rollup("Test A") 131 | User.rollup("Test B", interval: "week") 132 | User.rollup("Test C") 133 | expected = [ 134 | {name: "Test A", interval: "day"}, 135 | {name: "Test B", interval: "week"}, 136 | {name: "Test C", interval: "day"} 137 | ] 138 | assert_equal expected, Rollup.list 139 | end 140 | end 141 | -------------------------------------------------------------------------------- /test/rollup_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class RollupTest < Minitest::Test 4 | def test_attributes 5 | User.create! 6 | User.rollup("Test") 7 | 8 | assert_equal 1, Rollup.count 9 | 10 | rollup = Rollup.last 11 | assert_equal "Test", rollup.name 12 | assert_equal "day", rollup.interval 13 | assert_equal now.to_date, rollup.time 14 | assert_empty rollup.dimensions if dimensions_supported? 15 | assert_equal 1, rollup.value 16 | end 17 | 18 | def test_date 19 | today = Date.today 20 | User.create!(joined_on: today) 21 | User.rollup("Test", column: :joined_on, time_zone: false) 22 | 23 | rollup = Rollup.last 24 | assert_equal today, rollup.time 25 | assert_equal today, Rollup.series("Test").keys.first 26 | end 27 | 28 | def test_rename 29 | create_users 30 | User.rollup("Test") 31 | Rollup.rename("Test", "New") 32 | assert_equal 3, Rollup.where(name: "New").count 33 | assert_equal 0, Rollup.where(name: "Test").count 34 | end 35 | 36 | def test_time_cast 37 | create_users 38 | User.rollup("Test") 39 | assert_equal now.to_date, Rollup.last.time 40 | end 41 | 42 | def test_inspect 43 | create_users 44 | User.rollup("Test") 45 | assert_match "time: \"#{now.to_date}\"", Rollup.last.inspect 46 | end 47 | 48 | # uses date in upsert, no time 49 | def test_upsert_date 50 | create_users 51 | User.rollup("Test") 52 | if ActiveRecord::VERSION::MAJOR >= 8 53 | assert_match "'#{now.to_date} 00:00:00'", $sql.find { |s| s =~ /ON (CONFLICT|DUPLICATE KEY)/i } 54 | else 55 | assert_match "'#{now.to_date}'", $sql.find { |s| s =~ /ON (CONFLICT|DUPLICATE KEY)/i } 56 | end 57 | end 58 | 59 | def test_rollup_rollup 60 | User.create! 61 | User.rollup("Test") 62 | User.rollup("Other") 63 | Rollup.where(name: "Test", interval: "day").rollup("New", interval: "month", time_zone: false) 64 | assert_equal 1, Rollup.series("New", interval: "month").values[0] 65 | end 66 | 67 | def test_connection_leasing 68 | ActiveRecord::Base.connection_handler.clear_active_connections! 69 | assert_nil ActiveRecord::Base.connection_pool.active_connection? 70 | ActiveRecord::Base.connection_pool.with_connection do 71 | User.rollup("Test") 72 | Rollup.series("Test") 73 | end 74 | assert_nil ActiveRecord::Base.connection_pool.active_connection? 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /test/rollups_generator_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | require "rails/generators/test_case" 4 | require "generators/rollups_generator" 5 | 6 | class RollupsGeneratorTest < Rails::Generators::TestCase 7 | tests RollupsGenerator 8 | destination File.expand_path("../tmp", __dir__) 9 | setup :prepare_destination 10 | 11 | def test_works 12 | run_generator 13 | assert_migration "db/migrate/create_rollups.rb", /create_table :rollups/ 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /test/support/active_record.rb: -------------------------------------------------------------------------------- 1 | require "active_record" 2 | 3 | $adapter = ENV["ADAPTER"] || "postgresql" 4 | puts "Using #{$adapter}" 5 | 6 | def postgresql? 7 | $adapter == "postgresql" 8 | end 9 | 10 | def mysql? 11 | $adapter == "mysql" || $adapter == "trilogy" 12 | end 13 | 14 | def trilogy? 15 | $adapter == "trilogy" 16 | end 17 | 18 | def sqlite? 19 | $adapter == "sqlite" 20 | end 21 | 22 | def dimensions_supported? 23 | postgresql? 24 | end 25 | 26 | logger = ActiveSupport::Logger.new(ENV["VERBOSE"] ? STDOUT : nil) 27 | ActiveRecord::Migration.verbose = false unless ENV["VERBOSE"] 28 | 29 | Time.zone = sqlite? ? "Etc/UTC" : "Eastern Time (US & Canada)" 30 | 31 | # rails does this in activerecord/lib/active_record/railtie.rb 32 | ActiveRecord.default_timezone = :utc 33 | ActiveRecord::Base.time_zone_aware_attributes = true 34 | 35 | if ActiveRecord::VERSION::STRING.to_f >= 7.2 36 | ActiveRecord::Base.attributes_for_inspect = :all 37 | end 38 | 39 | if ActiveRecord::VERSION::STRING.to_f == 8.0 40 | ActiveSupport.to_time_preserves_timezone = :zone 41 | elsif ActiveRecord::VERSION::STRING.to_f == 7.2 42 | ActiveSupport.to_time_preserves_timezone = true 43 | end 44 | 45 | ActiveRecord::Base.logger = logger 46 | 47 | if postgresql? 48 | ActiveRecord::Base.establish_connection adapter: "postgresql", database: "rollup_test" 49 | elsif trilogy? 50 | ActiveRecord::Base.establish_connection adapter: "trilogy", database: "rollup_test", host: "127.0.0.1" 51 | elsif mysql? 52 | ActiveRecord::Base.establish_connection adapter: "mysql2", database: "rollup_test" 53 | else 54 | ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:" 55 | end 56 | 57 | ActiveRecord::Schema.define do 58 | create_table :posts, force: true do |t| 59 | t.references :user 60 | t.datetime :posted_at 61 | end 62 | 63 | create_table :rollups, force: true do |t| 64 | t.string :name, null: false 65 | t.string :interval, null: false 66 | t.datetime :time, null: false 67 | t.jsonb :dimensions, null: false, default: {} if dimensions_supported? 68 | t.float :value 69 | end 70 | if dimensions_supported? 71 | add_index :rollups, [:name, :interval, :time, :dimensions], unique: true 72 | else 73 | add_index :rollups, [:name, :interval, :time], unique: true 74 | end 75 | 76 | create_table :users, force: true do |t| 77 | t.string :os 78 | t.string :browser 79 | t.jsonb :properties if dimensions_supported? 80 | t.integer :visits 81 | t.date :joined_on 82 | t.timestamps 83 | end 84 | end 85 | 86 | class Post < ActiveRecord::Base 87 | belongs_to :user 88 | 89 | self.rollup_column = :posted_at 90 | end 91 | 92 | class User < ActiveRecord::Base 93 | has_many :posts 94 | end 95 | 96 | $sql = [] 97 | ActiveSupport::Notifications.subscribe("sql.active_record") do |_, _, _, _, payload| 98 | $sql << payload[:sql] 99 | end 100 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require "bundler/setup" 2 | Bundler.require(:default) 3 | require "minitest/autorun" 4 | require "minitest/pride" 5 | 6 | require_relative "support/active_record" 7 | 8 | class Minitest::Test 9 | def setup 10 | User.delete_all 11 | Post.delete_all 12 | Rollup.delete_all 13 | $sql = [] 14 | end 15 | 16 | # freeze time 17 | def now 18 | @now ||= Time.current 19 | end 20 | 21 | def create_users(interval: "day", browser: [], os: [], visits: []) 22 | step = interval_step(interval) 23 | 3.times do |i| 24 | created_at = now.dup 25 | i.times do 26 | created_at -= step 27 | end 28 | User.create!( 29 | browser: browser[i], 30 | os: os[i], 31 | visits: visits[i], 32 | created_at: created_at 33 | ) 34 | end 35 | end 36 | 37 | def interval_step(interval) 38 | case interval 39 | when "1s" 40 | 1.second 41 | when "30s" 42 | 30.seconds 43 | when "1m" 44 | 1.minute 45 | when "5m" 46 | 5.minutes 47 | when "quarter" 48 | 3.months 49 | else 50 | 1.send(interval) 51 | end 52 | end 53 | end 54 | --------------------------------------------------------------------------------