├── .rspec ├── .travis.yml ├── lib ├── fast_statistics │ └── version.rb └── fast_statistics.rb ├── bin ├── setup ├── console ├── rake └── rspec ├── .clang-format ├── .gitignore ├── benchmark ├── helpers.rb ├── benchmark.rb └── base.rb ├── ext └── fast_statistics │ ├── fast_statistics.h │ ├── extconf.rb │ ├── debug.h │ ├── array_2d.h │ ├── fast_statistics.cpp │ └── array_2d.cpp ├── Gemfile ├── Rakefile ├── LICENSE.txt ├── fast_statistics.gemspec ├── spec ├── spec_helper.rb └── fast_statistics_spec.rb └── README.md /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper 2 | --format documentation 3 | --fail-fast 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.5 4 | - 2.6 5 | - 2.7 6 | -------------------------------------------------------------------------------- /lib/fast_statistics/version.rb: -------------------------------------------------------------------------------- 1 | module FastStatistics 2 | VERSION = "0.2.1" 3 | end 4 | -------------------------------------------------------------------------------- /lib/fast_statistics.rb: -------------------------------------------------------------------------------- 1 | require "fast_statistics/version" 2 | require "fast_statistics/fast_statistics" 3 | 4 | module FastStatistics 5 | end 6 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | BasedOnStyle: Mozilla 3 | ColumnLimit: 100 4 | TabWidth: 2 5 | UseTab: Never 6 | BreakBeforeBraces: Linux 7 | AllowShortIfStatementsOnASingleLine: true 8 | AlignAfterOpenBracket: AlwaysBreak 9 | AllowAllArgumentsOnNextLine: true 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | 10 | Gemfile.lock 11 | 12 | *.so 13 | *.o 14 | compile_flags.txt 15 | compile_commands.json 16 | ext/fast_statistics/Makefile 17 | ext/fast_statistics/mkmf.log 18 | .cache 19 | 20 | squiggles.txt 21 | README.md.html 22 | -------------------------------------------------------------------------------- /benchmark/helpers.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Helpers 4 | def percentile(p, arr, len) 5 | return arr[len - 1] if p == 100 6 | rank = p / 100.0 * (len - 1) 7 | lower = arr[rank.floor] 8 | upper = arr[rank.floor + 1] 9 | lower + (upper - lower) * (rank - rank.floor) 10 | end 11 | end 12 | 13 | -------------------------------------------------------------------------------- /ext/fast_statistics/fast_statistics.h: -------------------------------------------------------------------------------- 1 | #ifndef FAST_STATISTICS_H 2 | #define FAST_STATISTICS_H 3 | 4 | #include 5 | #ifdef HAVE_XMMINTRIN_H 6 | #include 7 | #endif 8 | 9 | #include "debug.h" 10 | 11 | #define rb_sym(str) ID2SYM(rb_intern(str)) 12 | #define UNWRAP_DFLOAT(obj, var) TypedData_Get_Struct((obj), void*, &dfloat_wrapper, (var)); 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "fast_statistics" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start(__FILE__) 15 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | # Specify your gem's dependencies in fast_statistics.gemspec 4 | gemspec 5 | 6 | gem "rake", "~> 13.0" 7 | gem "rake-compiler", "~> 1.1" 8 | gem "rspec", "~> 3.0" 9 | 10 | # Dependencies for running benchmarks 11 | gem "benchmark-ips" 12 | gem "terminal-table" 13 | gem "descriptive_statistics" 14 | gem "ruby_native_statistics" 15 | gem "numo-narray" 16 | gem "nmatrix" 17 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rake/extensiontask" 3 | 4 | task :default => :spec 5 | 6 | # Compile 7 | Rake::ExtensionTask.new "fast_statistics" do |ext| 8 | ext.lib_dir = "lib/fast_statistics" 9 | end 10 | 11 | # Rspec 12 | begin 13 | require 'rspec/core/rake_task' 14 | RSpec::Core::RakeTask.new(:spec, [:spec] => [:clean, :compile]) 15 | rescue LoadError 16 | end 17 | 18 | # Benchmark 19 | task :benchmark => [:clean, :compile] do 20 | require_relative "./benchmark/benchmark" 21 | bench = DescriptiveStatsBenchmark 22 | bench.compare_results! 23 | bench.benchmark_ips! 24 | end 25 | -------------------------------------------------------------------------------- /ext/fast_statistics/extconf.rb: -------------------------------------------------------------------------------- 1 | require "mkmf" 2 | 3 | # Enable debug compile using DEBUG env var 4 | if ENV["DEBUG"] 5 | puts "Compiling in debug mode..." 6 | CONFIG["debugflags"] = "-g" 7 | CONFIG["optflags"] = "-O0" 8 | $defs << "-DDEBUG" 9 | end 10 | 11 | # Compile with C++11 12 | $CXXFLAGS += " -std=c++11 " 13 | 14 | # Disable warnings 15 | [ 16 | / -Wdeclaration-after-statement/, 17 | / -Wimplicit-int/, 18 | / -Wimplicit-function-declaration/, 19 | ].each do |flag| 20 | CONFIG["warnflags"].slice!(flag) 21 | end 22 | 23 | have_header("xmmintrin.h") 24 | create_makefile("fast_statistics/fast_statistics") 25 | -------------------------------------------------------------------------------- /ext/fast_statistics/debug.h: -------------------------------------------------------------------------------- 1 | #ifdef DEBUG 2 | #ifndef DEBUG_H 3 | #define DEBUG_H 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #define CONCAT_(a, b) a##b 10 | #define CONCAT(a,b) CONCAT_(a, b) 11 | #define PROFILE DebugTimer CONCAT(Timer, __COUNTER__)(__func__, __LINE__); 12 | 13 | struct DebugTimer { 14 | char* name; 15 | unsigned long long counter; 16 | 17 | DebugTimer(const char* function_name, int line_number) { 18 | name = (char*)malloc(200 * sizeof(char)); 19 | 20 | strcpy(name, function_name); 21 | strcat(name, "_"); 22 | snprintf(name + strlen(name), 4, "%d", line_number); 23 | 24 | counter = __rdtsc(); 25 | } 26 | 27 | ~DebugTimer() { 28 | printf("\n%30s:\t %-10llu", name, __rdtsc() - counter); 29 | free(name); 30 | fflush(stdout); 31 | } 32 | }; 33 | #endif 34 | #else 35 | #define PROFILE(); 36 | #endif 37 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'rake' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require "pathname" 12 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path("../bundle", __FILE__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require "rubygems" 27 | require "bundler/setup" 28 | 29 | load Gem.bin_path("rake", "rake") 30 | -------------------------------------------------------------------------------- /bin/rspec: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'rspec' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require "pathname" 12 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path("../bundle", __FILE__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require "rubygems" 27 | require "bundler/setup" 28 | 29 | load Gem.bin_path("rspec-core", "rspec") 30 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 Martin Nyaga 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /fast_statistics.gemspec: -------------------------------------------------------------------------------- 1 | require_relative 'lib/fast_statistics/version' 2 | 3 | Gem::Specification.new do |spec| 4 | spec.name = "fast_statistics" 5 | spec.version = FastStatistics::VERSION 6 | spec.authors = ["Martin Nyaga"] 7 | spec.email = ["martin@martinnyaga.com"] 8 | 9 | spec.summary = %q{Fast computation of descriptive statistics in ruby} 10 | spec.description = %q{Fast computation of descriptive statistics in ruby using native code and SIMD} 11 | spec.homepage = "https://github.com/martin-nyaga/ruby-ffi-simd" 12 | spec.license = "MIT" 13 | spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0") 14 | 15 | spec.metadata["homepage_uri"] = spec.homepage 16 | spec.metadata["source_code_uri"] = "https://github.com/martin-nyaga/fast_statistics" 17 | spec.metadata["changelog_uri"] = "https://github.com/martin-nyaga/fast_statistics" 18 | 19 | # Specify which files should be added to the gem when it is released. 20 | # The `git ls-files -z` loads the files in the RubyGem that have been added into git. 21 | spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do 22 | `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 23 | end 24 | spec.require_paths = ["lib"] 25 | 26 | spec.extensions = %w[ext/fast_statistics/extconf.rb] 27 | end 28 | -------------------------------------------------------------------------------- /ext/fast_statistics/array_2d.h: -------------------------------------------------------------------------------- 1 | #ifndef ARRAY_2D_H 2 | #define ARRAY_2D_H 3 | #include 4 | #include "ruby.h" 5 | 6 | #ifdef HAVE_XMMINTRIN_H 7 | #include 8 | #define MM_GET_INDEX(packed, index) *(((double*)&packed) + index); 9 | #endif 10 | 11 | namespace array_2d 12 | { 13 | 14 | struct Stats { 15 | double min; 16 | double max; 17 | double mean; 18 | double median; 19 | double q1; 20 | double q3; 21 | double standard_deviation; 22 | 23 | Stats() 24 | { 25 | min = 0.0, max = 0.0, mean = 0.0, median = 0.0, q1 = 0.0, q3 = 0.0, standard_deviation = 0.0; 26 | }; 27 | }; 28 | 29 | class DFloat 30 | { 31 | inline double* base_ptr(int col) { return entries + (col * rows); } 32 | inline void sort(double* col); 33 | inline double percentile(double* col, double pct); 34 | inline double sum(double* col); 35 | inline double standard_deviation(double* col, double mean); 36 | 37 | #ifdef HAVE_XMMINTRIN_H 38 | inline double safe_entry(int col, int row); 39 | inline void sort_columns(int start_col, int pack_size); 40 | inline __m128d percentile_packed(int start_col, float pct); 41 | inline __m128d pack(int start_col, int row); 42 | #endif 43 | 44 | public: 45 | int cols; 46 | int rows; 47 | bool data_initialized; 48 | double* entries; 49 | Stats* stats; 50 | 51 | DFloat(VALUE ruby_arr, bool initialize_data); 52 | ~DFloat(); 53 | 54 | Stats* descriptive_statistics(); 55 | 56 | #ifdef HAVE_XMMINTRIN_H 57 | Stats* descriptive_statistics_packed(); 58 | #endif 59 | }; 60 | } // namespace array_2d 61 | #endif 62 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 4 | RSpec.configure do |config| 5 | config.expect_with :rspec do |expectations| 6 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 7 | end 8 | 9 | config.mock_with :rspec do |mocks| 10 | mocks.verify_partial_doubles = true 11 | end 12 | 13 | config.filter_run :focus => true 14 | config.run_all_when_everything_filtered = true 15 | end 16 | 17 | require 'rspec/expectations' 18 | require 'pp' 19 | 20 | RSpec::Matchers.define :have_same_statistics_values_as do |expected| 21 | match do |actual| 22 | expect(expected.length).to eq(actual.length) 23 | 24 | expected.each_with_index do |stats, index| 25 | actual[index].each do |(k, v)| 26 | @index = index 27 | @key = k 28 | @expected_value = stats[k] 29 | @actual_value = v 30 | expect(v).to be_within(threshold).of(@expected_value) 31 | end 32 | end 33 | end 34 | 35 | def threshold 36 | @threshold || default_threshold 37 | end 38 | 39 | def default_threshold 40 | 10e-9 41 | end 42 | 43 | chain :within_threshold do |threshold| 44 | @threshold = threshold 45 | end 46 | 47 | failure_message do |actual| 48 | <<~MSG 49 | For #{@key} at index #{@index} 50 | expected: #{@actual_value} to be within #{threshold} of #{@expected_value} 51 | 52 | expected array: 53 | #{expected.pretty_inspect} 54 | 55 | actual array: 56 | #{actual.pretty_inspect} 57 | MSG 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /benchmark/benchmark.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/setup" 4 | 5 | require_relative "./base" 6 | require_relative "./helpers" 7 | 8 | require "fast_statistics" 9 | require "descriptive_statistics/safe" 10 | require "ruby_native_statistics" 11 | require "numo/narray" 12 | 13 | class DescriptiveStatsBenchmark < BaseBenchmark 14 | class << self 15 | include Helpers 16 | end 17 | 18 | benchmark "descriptive_statistics" do |data| 19 | data.map do |arr| 20 | stats = DescriptiveStatistics::Stats.new(arr) 21 | { 22 | mean: stats.mean, 23 | min: stats.min, 24 | max: stats.max, 25 | median: stats.median, 26 | q1: stats.percentile(25), 27 | q3: stats.percentile(75), 28 | standard_deviation: stats.standard_deviation 29 | } 30 | end 31 | end 32 | 33 | benchmark "Custom ruby" do |data| 34 | data.map do |arr| 35 | arr.sort! 36 | 37 | min = arr.first 38 | max = arr.last 39 | length = arr.length 40 | median = percentile(50, arr, length) 41 | q1 = percentile(25, arr, length) 42 | q3 = percentile(75, arr, length) 43 | sum = arr.inject(0) { |sum, x| sum + x} 44 | 45 | mean = sum / length 46 | variance = arr.inject(0) { |var, x| var += ((x - mean) ** 2) / length } 47 | standard_deviation = Math.sqrt(variance) 48 | { 49 | mean: mean, 50 | min: min, 51 | max: max, 52 | median: median, 53 | q1: q1, 54 | q3: q3, 55 | standard_deviation: standard_deviation 56 | } 57 | end 58 | end 59 | 60 | benchmark "narray" do |data| 61 | data.map do |arr| 62 | narr = Numo::DFloat[arr] 63 | narr.sort 64 | min = narr[0] 65 | length = arr.length 66 | max = narr[length - 1] 67 | median = percentile(50, narr, length) 68 | q1 = percentile(25, narr, length) 69 | q3 = percentile(75, narr, length) 70 | mean = narr.mean 71 | variance = 0 72 | narr.each { |x| variance += ((x - mean) ** 2) / length } 73 | standard_deviation = Math.sqrt(variance) 74 | { 75 | mean: mean, 76 | min: min, 77 | max: max, 78 | median: median, 79 | q1: q1, 80 | q3: q3, 81 | standard_deviation: standard_deviation 82 | } 83 | end 84 | end 85 | 86 | benchmark "ruby_native_statistics" do |data| 87 | data.map do |arr| 88 | { 89 | mean: arr.mean, 90 | min: arr.min, 91 | max: arr.max, 92 | median: arr.median, 93 | q1: arr.percentile(0.25), 94 | q3: arr.percentile(0.75), 95 | standard_deviation: arr.stdevp 96 | } 97 | end 98 | end 99 | 100 | benchmark "FastStatistics" do |data| 101 | FastStatistics::Array2D.new(data).descriptive_statistics 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /spec/fast_statistics_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'fast_statistics' 4 | 5 | describe FastStatistics::Array2D do 6 | let(:data) do 7 | [ 8 | [0.6269, 0.3783, 0.1477, 0.2374], 9 | [0.4209, 0.1055, 0.8000, 0.2023], 10 | [0.1124, 0.1021, 0.1936, 0.8566], 11 | [0.6454, 0.5362, 0.4567, 0.8309], 12 | [0.4828, 0.1572, 0.5706, 0.4085], 13 | [0.5594, 0.0979, 0.4078, 0.5885], 14 | [0.8659, 0.5346, 0.5566, 0.6166], 15 | [0.7256, 0.5841, 0.8546, 0.3918] 16 | ] 17 | end 18 | 19 | let(:expected_stats) do 20 | [ 21 | { min: 0.1477, max: 0.6269, mean: 0.347575, median: 0.30785, q1: 0.214975, q3: 0.440450, standard_deviation: 0.1810076155165853 }, 22 | { min: 0.1055, max: 0.8000, mean: 0.382175, median: 0.31160, q1: 0.178100, q3: 0.515675, standard_deviation: 0.2669182587890907 }, 23 | { min: 0.1021, max: 0.8566, mean: 0.316175, median: 0.15300, q1: 0.109825, q3: 0.359350, standard_deviation: 0.3140207666301705 }, 24 | { min: 0.4567, max: 0.8309, mean: 0.617300, median: 0.59080, q1: 0.516325, q3: 0.691775, standard_deviation: 0.1403425630377327 }, 25 | { min: 0.1572, max: 0.5706, mean: 0.404775, median: 0.44565, q1: 0.345675, q3: 0.504750, standard_deviation: 0.1540236081742016 }, 26 | { min: 0.0979, max: 0.5885, mean: 0.413400, median: 0.48360, q1: 0.330325, q3: 0.566675, standard_deviation: 0.1946455881852964 }, 27 | { min: 0.5346, max: 0.8659, mean: 0.643425, median: 0.58660, q1: 0.551100, q3: 0.678925, standard_deviation: 0.1319054277692923 }, 28 | { min: 0.3918, max: 0.8546, mean: 0.639025, median: 0.65485, q1: 0.536025, q3: 0.757850, standard_deviation: 0.1718318709523935 } 29 | ] 30 | end 31 | 32 | context "with 8 variables" do 33 | subject { FastStatistics::Array2D.new(data) } 34 | 35 | it "#descriptive_statistics works" do 36 | stats = subject.descriptive_statistics 37 | expect(stats).to have_same_statistics_values_as(expected_stats) 38 | end 39 | end 40 | 41 | context "with 1 variable" do 42 | subject { FastStatistics::Array2D.new(data.first(1)) } 43 | it "#descriptive_statistics works" do 44 | stats = subject.descriptive_statistics 45 | expect(stats).to have_same_statistics_values_as(expected_stats.first(1)) 46 | end 47 | end 48 | 49 | context "with an odd number of variables" do 50 | subject { FastStatistics::Array2D.new(data.first(5)) } 51 | it "#descriptive_statistics works" do 52 | stats = subject.descriptive_statistics 53 | expect(stats).to have_same_statistics_values_as(expected_stats.first(5)) 54 | end 55 | end 56 | 57 | context "with incorrect initialization" do 58 | it "should throw a type error" do 59 | expect { FastStatistics::Array2D.new([1, 2, 3]) }.to raise_error(TypeError) 60 | expect { FastStatistics::Array2D.new("hello world") }.to raise_error(TypeError) 61 | end 62 | end 63 | 64 | context "simd_enabled?" do 65 | it "allows to check if simd is enabled" do 66 | expect { FastStatistics.simd_enabled? }.not_to raise_error 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /benchmark/base.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "benchmark" 4 | require "benchmark/ips" 5 | require "terminal-table" 6 | 7 | class BaseBenchmark 8 | class << self 9 | @@tests = [] 10 | 11 | def benchmark(name, &block) 12 | @@tests.push(TestCase.new(name, block)) 13 | end 14 | 15 | def tests 16 | @@tests 17 | end 18 | 19 | def compare_results!(data_points: 10, precision: 6) 20 | data = generate_data(data_points) 21 | puts("Comparing calculated statistics with #{format_number(data_points)} values for #{data.length} variables...") 22 | 23 | test_results = tests.map { |test| test.run(data) } 24 | 25 | # Uncomment to print results 26 | # test_results.zip(tests) do |results, test| 27 | # print_results(test.name, results, precision) 28 | # end 29 | if assert_values_within_delta(test_results, 10 ** -precision) 30 | puts("Test passed, results are equal to #{precision} decimal places!") 31 | puts 32 | end 33 | 34 | rescue TestFailure => e 35 | puts("Test results did not match!") 36 | exit(1) 37 | end 38 | 39 | def benchmark!(data_points: 100_000, variables_count: 12) 40 | data = generate_data(data_points, variables_count) 41 | puts("Benchmarking with #{format_number(data_points)} values for #{data.length} variables...") 42 | 43 | ::Benchmark.bmbm do |x| 44 | tests.each do |test| 45 | x.report(test.name) do 46 | test.run(data) 47 | end 48 | end 49 | end 50 | end 51 | 52 | def benchmark_ips!(data_points: 100_000, variables_count: 12) 53 | data = generate_data(data_points, variables_count) 54 | puts("Benchmarking with #{format_number(data_points)} values for #{data.length} variables...") 55 | 56 | ::Benchmark.ips do |x| 57 | tests.each do |test| 58 | x.report(test.name) do 59 | test.run(data) 60 | end 61 | 62 | x.compare! 63 | end 64 | end 65 | end 66 | 67 | private 68 | 69 | def generate_data(length, variables = 8) 70 | data = (0..(variables - 1)).map { (0..(length - 1)).map { rand } } 71 | end 72 | 73 | def print_results(title, results, precision) 74 | headers = results[0].keys 75 | values = results.map { |r| r.values.map { |v| "%.#{precision}f" % v } } 76 | table = Terminal::Table.new(headings: headers, rows: values) 77 | puts(title + ":") 78 | puts(table) 79 | end 80 | 81 | def assert_values_within_delta(values, delta) 82 | values.combination(2).each do |expected, actual| 83 | unless expected.length == actual.length 84 | raise TestFailure, "Results don't match!" 85 | end 86 | 87 | expected.each_with_index do |expected_result, i| 88 | actual_result = actual[i] 89 | 90 | if actual_result.is_a?(Hash) && expected_result.is_a?(Hash) 91 | expected_result.each do |k, _v| 92 | assert_in_delta(actual_result[k], expected_result[k], delta) 93 | end 94 | else 95 | 96 | assert_in_delta(actual_result, expected_result, delta) 97 | end 98 | end 99 | end 100 | 101 | true 102 | end 103 | 104 | def assert_in_delta(expected, actual, delta) 105 | unless (expected - actual).abs < delta 106 | raise TestFailure, "Results don't match!" 107 | end 108 | 109 | true 110 | end 111 | 112 | def format_number(number) 113 | number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse 114 | end 115 | 116 | TestCase = Struct.new(:name, :block) do 117 | def run(data) 118 | block.call(data) 119 | end 120 | end 121 | 122 | class TestFailure < StandardError 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /ext/fast_statistics/fast_statistics.cpp: -------------------------------------------------------------------------------- 1 | #include "fast_statistics.h" 2 | #include "array_2d.h" 3 | 4 | using namespace array_2d; 5 | 6 | static VALUE mFastStatistics; 7 | static VALUE cArray2D; 8 | 9 | // Helper 10 | VALUE 11 | build_results_hashes(Stats* stats, int num_variables) 12 | { 13 | VALUE a_results = rb_ary_new(); 14 | 15 | VALUE s_min = rb_sym("min"); 16 | VALUE s_max = rb_sym("max"); 17 | VALUE s_mean = rb_sym("mean"); 18 | VALUE s_median = rb_sym("median"); 19 | VALUE s_q1 = rb_sym("q1"); 20 | VALUE s_q3 = rb_sym("q3"); 21 | VALUE s_standard_deviation = rb_sym("standard_deviation"); 22 | 23 | for (int i = 0; i < num_variables; i++) { 24 | VALUE h_result = rb_hash_new(); 25 | Stats var_stats = stats[i]; 26 | 27 | rb_hash_aset(h_result, s_min, DBL2NUM(var_stats.min)); 28 | rb_hash_aset(h_result, s_max, DBL2NUM(var_stats.max)); 29 | rb_hash_aset(h_result, s_mean, DBL2NUM(var_stats.mean)); 30 | rb_hash_aset(h_result, s_median, DBL2NUM(var_stats.median)); 31 | rb_hash_aset(h_result, s_q1, DBL2NUM(var_stats.q1)); 32 | rb_hash_aset(h_result, s_q3, DBL2NUM(var_stats.q3)); 33 | rb_hash_aset(h_result, s_standard_deviation, DBL2NUM(var_stats.standard_deviation)); 34 | 35 | rb_ary_push(a_results, h_result); 36 | } 37 | 38 | return a_results; 39 | } 40 | 41 | // Common 42 | void 43 | free_wrapped_array(void* dfloat) 44 | { 45 | ((DFloat*)dfloat)->~DFloat(); 46 | free(dfloat); 47 | } 48 | 49 | size_t 50 | wrapped_array_size(const void* data) 51 | { 52 | return sizeof(DFloat); 53 | } 54 | 55 | static rb_data_type_t dfloat_wrapper = [] { 56 | rb_data_type_t wrapper{}; 57 | wrapper.wrap_struct_name = "dfloat"; 58 | wrapper.function = { dmark : NULL, dfree : free_wrapped_array, dsize : wrapped_array_size }; 59 | wrapper.data = NULL; 60 | wrapper.flags = RUBY_TYPED_FREE_IMMEDIATELY; 61 | return wrapper; 62 | }(); 63 | 64 | VALUE 65 | cArray2D_alloc(VALUE self) 66 | { 67 | void* dfloat = (void*)malloc(sizeof(DFloat)); 68 | 69 | return TypedData_Wrap_Struct(self, &dfloat_wrapper, dfloat); 70 | } 71 | 72 | /* 73 | * def initialize(arrays) 74 | */ 75 | VALUE 76 | cArray2D_initialize(VALUE self, VALUE arrays) 77 | { 78 | // Initialize dfloat structure to store Dfloat in type wrapper 79 | void* dfloat; 80 | UNWRAP_DFLOAT(self, dfloat); 81 | 82 | // Type-check 2D array 83 | if (TYPE(arrays) != T_ARRAY) { 84 | new (dfloat) DFloat(arrays, false); 85 | Check_Type(arrays, T_ARRAY); 86 | return false; 87 | } 88 | 89 | if (TYPE(rb_ary_entry(arrays, 0)) != T_ARRAY) { 90 | new (dfloat) DFloat(arrays, false); 91 | Check_Type(rb_ary_entry(arrays, 0), T_ARRAY); 92 | return false; 93 | } 94 | 95 | new (dfloat) DFloat(arrays, true); 96 | return self; 97 | } 98 | 99 | //{{{ Unpacked 100 | VALUE 101 | simd_disabled(VALUE self) 102 | { 103 | return Qfalse; 104 | } 105 | 106 | /* 107 | * Unpacked descriptive statistics 108 | * 109 | * def descriptive_statistics 110 | */ 111 | VALUE 112 | cArray2D_descriptive_statistics_unpacked(VALUE self) 113 | { 114 | void* dfloat_untyped; 115 | UNWRAP_DFLOAT(self, dfloat_untyped); 116 | 117 | DFloat* dfloat = ((DFloat*)dfloat_untyped); 118 | Stats* stats = dfloat->descriptive_statistics(); 119 | return build_results_hashes(stats, dfloat->cols); 120 | } 121 | //}}} 122 | 123 | //{{{ Packed 124 | #ifdef HAVE_XMMINTRIN_H 125 | extern "C" VALUE 126 | simd_enabled(VALUE self) 127 | { 128 | return Qtrue; 129 | } 130 | 131 | /* 132 | * Packed descriptive statistics 133 | * 134 | * def descriptive_statistics 135 | */ 136 | VALUE 137 | cArray2D_descriptive_statistics_packed(VALUE self) 138 | { 139 | void* dfloat_untyped; 140 | UNWRAP_DFLOAT(self, dfloat_untyped); 141 | 142 | DFloat* dfloat = ((DFloat*)dfloat_untyped); 143 | Stats* stats = dfloat->descriptive_statistics_packed(); 144 | return build_results_hashes(stats, dfloat->cols); 145 | } 146 | #endif 147 | //}}} 148 | 149 | extern "C" void 150 | Init_fast_statistics(void) 151 | { 152 | mFastStatistics = rb_define_module("FastStatistics"); 153 | cArray2D = rb_define_class_under(mFastStatistics, "Array2D", rb_cData); 154 | rb_define_alloc_func(cArray2D, cArray2D_alloc); 155 | rb_define_method(cArray2D, "initialize", RUBY_METHOD_FUNC(cArray2D_initialize), 1); 156 | 157 | #ifdef HAVE_XMMINTRIN_H 158 | rb_define_singleton_method(mFastStatistics, "simd_enabled?", RUBY_METHOD_FUNC(simd_enabled), 0); 159 | rb_define_method( 160 | cArray2D, 161 | "descriptive_statistics", 162 | RUBY_METHOD_FUNC(cArray2D_descriptive_statistics_packed), 163 | 0); 164 | #else 165 | rb_define_singleton_method(mFastStatistics, "simd_enabled?", RUBY_METHOD_FUNC(simd_disabled), 0); 166 | rb_define_method( 167 | cArray2D, 168 | "descriptive_statistics", 169 | RUBY_METHOD_FUNC(cArray2D_descriptive_statistics_unpacked), 170 | 0); 171 | #endif 172 | } 173 | -------------------------------------------------------------------------------- /ext/fast_statistics/array_2d.cpp: -------------------------------------------------------------------------------- 1 | #include "array_2d.h" 2 | #include "debug.h" 3 | 4 | namespace array_2d 5 | { 6 | 7 | DFloat::~DFloat() 8 | { 9 | if (data_initialized) { 10 | free(entries); 11 | delete[] stats; 12 | } 13 | } 14 | 15 | DFloat::DFloat(VALUE arrays, bool initialize_data) 16 | { 17 | data_initialized = initialize_data; 18 | 19 | if (initialize_data) { 20 | cols = rb_array_len(arrays); 21 | rows = rb_array_len(rb_ary_entry(arrays, 0)); 22 | entries = (double*)malloc(cols * rows * sizeof(double)); 23 | stats = new Stats[cols]; 24 | 25 | for (int j = 0; j < cols; j++) { 26 | for (int i = 0; i < rows; i++) { 27 | entries[j * rows + i] = (double)NUM2DBL(rb_ary_entry(rb_ary_entry(arrays, j), i)); 28 | } 29 | } 30 | } else { 31 | cols = 0; 32 | rows = 0; 33 | entries = NULL; 34 | stats = NULL; 35 | } 36 | } 37 | 38 | inline void 39 | DFloat::sort(double* col) 40 | { 41 | std::sort(col, col + rows); 42 | } 43 | 44 | inline double 45 | DFloat::percentile(double* col, double pct) 46 | { 47 | if (pct == 1.0) return col[rows - 1]; 48 | double rank = pct * (double)(rows - 1); 49 | int floored_rank = floor(rank); 50 | double lower = col[floored_rank]; 51 | double upper = col[floored_rank + 1]; 52 | return lower + (upper - lower) * (rank - floored_rank); 53 | } 54 | 55 | inline double 56 | DFloat::sum(double* col) 57 | { 58 | double sum = 0.0; 59 | for (int row = 0; row < rows; row++) { 60 | sum += col[row]; 61 | } 62 | return sum; 63 | } 64 | 65 | inline double 66 | DFloat::standard_deviation(double* col, double mean) 67 | { 68 | double variance = 0.0f; 69 | for (int i = 0; i < rows; i++) { 70 | double value = col[i]; 71 | double deviation = value - mean; 72 | double sqr_deviation = deviation * deviation; 73 | variance += (sqr_deviation / (double)rows); 74 | } 75 | double result = sqrt(variance); 76 | return result; 77 | } 78 | 79 | Stats* 80 | DFloat::descriptive_statistics() 81 | { 82 | if (!data_initialized) return stats; 83 | 84 | for (int col = 0; col < cols; col++) { 85 | Stats var_stats; 86 | double* col_arr = base_ptr(col); 87 | 88 | sort(col_arr); 89 | 90 | var_stats.min = col_arr[0]; 91 | var_stats.max = col_arr[rows - 1]; 92 | var_stats.median = percentile(col_arr, 0.5); 93 | var_stats.q1 = percentile(col_arr, 0.25); 94 | var_stats.q3 = percentile(col_arr, 0.75); 95 | double total = sum(col_arr); 96 | var_stats.mean = total / (double)rows; 97 | var_stats.standard_deviation = standard_deviation(col_arr, var_stats.mean); 98 | 99 | stats[col] = var_stats; 100 | } 101 | 102 | return stats; 103 | } 104 | 105 | #ifdef HAVE_XMMINTRIN_H 106 | inline double 107 | DFloat::safe_entry(int col, int row) 108 | { 109 | if (col < cols) { 110 | return *(base_ptr(col) + row); 111 | } else { 112 | return 0; 113 | } 114 | } 115 | 116 | inline void 117 | DFloat::sort_columns(int start_col, int pack_size) 118 | { 119 | for (int i = 0; i < pack_size; i++) { 120 | if ((start_col + i) < cols) { 121 | double* col_arr = base_ptr(start_col + i); 122 | sort(col_arr); 123 | } 124 | } 125 | } 126 | 127 | inline __m128d 128 | DFloat::pack(int start_col, int row) 129 | { 130 | __m128d packed = _mm_set_pd(safe_entry(start_col + 1, row), safe_entry(start_col + 0, row)); 131 | return packed; 132 | } 133 | 134 | inline __m128d 135 | DFloat::percentile_packed(int start_col, float pct) 136 | { 137 | if (pct == 1.0) return pack(start_col, rows - 1); 138 | double rank = pct * (double)(rows - 1); 139 | int floored_rank = floor(rank); 140 | __m128d lower = pack(start_col, floored_rank); 141 | __m128d upper = pack(start_col, floored_rank + 1); 142 | __m128d upper_minus_lower = _mm_sub_pd(upper, lower); 143 | __m128d rank_minus_floored_rank = _mm_sub_pd(_mm_set_pd1(rank), _mm_set_pd1((float)floored_rank)); 144 | return _mm_add_pd(lower, _mm_mul_pd(upper_minus_lower, rank_minus_floored_rank)); 145 | } 146 | 147 | Stats* 148 | DFloat::descriptive_statistics_packed() 149 | { 150 | stats = new Stats[cols]; 151 | if (!data_initialized) return stats; 152 | const int simd_pack_size = 2; 153 | 154 | __m128d lengths = _mm_set_pd1((double)rows); 155 | for (int col = 0; col < cols; col += simd_pack_size) { 156 | sort_columns(col, simd_pack_size); 157 | 158 | __m128d mins = pack(col, 0); 159 | __m128d maxes = pack(col, rows - 1); 160 | __m128d sums = _mm_setzero_pd(); 161 | for (int row_index = 0; row_index < rows; row_index++) { 162 | __m128d packed = pack(col, row_index); 163 | sums = _mm_add_pd(sums, packed); 164 | } 165 | __m128d means = _mm_div_pd(sums, lengths); 166 | 167 | __m128d medians = percentile_packed(col, 0.5f); 168 | __m128d q1s = percentile_packed(col, 0.25f); 169 | __m128d q3s = percentile_packed(col, 0.75f); 170 | 171 | __m128d variances = _mm_setzero_pd(); 172 | for (int row_index = 0; row_index < rows; row_index++) { 173 | __m128d packed = pack(col, row_index); 174 | __m128d deviation = _mm_sub_pd(packed, means); 175 | __m128d sqr_deviation = _mm_mul_pd(deviation, deviation); 176 | variances = _mm_add_pd(variances, _mm_div_pd(sqr_deviation, lengths)); 177 | } 178 | __m128d stdevs = _mm_sqrt_pd(variances); 179 | 180 | for (int simd_slot_index = 0; simd_slot_index < simd_pack_size; simd_slot_index++) { 181 | if ((col + simd_slot_index) < cols) { 182 | Stats var_stats; 183 | var_stats.min = MM_GET_INDEX(mins, simd_slot_index); 184 | var_stats.max = MM_GET_INDEX(maxes, simd_slot_index); 185 | var_stats.mean = MM_GET_INDEX(means, simd_slot_index); 186 | var_stats.median = MM_GET_INDEX(medians, simd_slot_index); 187 | var_stats.q1 = MM_GET_INDEX(q1s, simd_slot_index); 188 | var_stats.q3 = MM_GET_INDEX(q3s, simd_slot_index); 189 | var_stats.standard_deviation = MM_GET_INDEX(stdevs, simd_slot_index); 190 | 191 | stats[col + simd_slot_index] = var_stats; 192 | } 193 | } 194 | } 195 | 196 | return stats; 197 | } 198 | 199 | #endif 200 | } // namespace array_2d 201 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Fast Statistics :rocket: 2 | ![Build Status](https://travis-ci.com/Martin-Nyaga/fast_statistics.svg?branch=master) 3 | 4 | A high performance native ruby extension (written in C++) for computation of 5 | descriptive statistics. 6 | 7 | ## Overview 8 | This gem provides fast computation of descriptive statistics (min, max, mean, 9 | median, 1st and 3rd quartiles, population standard deviation) for a multivariate 10 | dataset (represented as a 2D array) in ruby. 11 | 12 | It is **~11x** faster than an optimal algorithm in hand-written ruby, and 13 | **~4.7x** faster than the next fastest available ruby gem or native extension 14 | (see [benchmarks](#benchmarks) below). 15 | 16 | ## Installation 17 | 18 | Add this line to your application's Gemfile: 19 | 20 | ```ruby 21 | gem 'fast_statistics' 22 | ``` 23 | 24 | And then execute: 25 | 26 | $ bundle install 27 | 28 | Or install it yourself as: 29 | 30 | $ gem install fast_statistics 31 | 32 | ## Usage 33 | 34 | Given you have some multivariate (2-dimensional) data: 35 | ```ruby 36 | data = [ 37 | [0.6269, 0.3783, 0.1477, 0.2374], 38 | [0.4209, 0.1055, 0.8000, 0.2023], 39 | [0.1124, 0.1021, 0.1936, 0.8566], 40 | [0.6454, 0.5362, 0.4567, 0.8309], 41 | [0.4828, 0.1572, 0.5706, 0.4085], 42 | [0.5594, 0.0979, 0.4078, 0.5885], 43 | [0.8659, 0.5346, 0.5566, 0.6166], 44 | [0.7256, 0.5841, 0.8546, 0.3918] 45 | ] 46 | ``` 47 | 48 | You can compute descriptive statistics for all the inner arrays as follows: 49 | 50 | ```ruby 51 | require "fast_statistics" 52 | 53 | FastStatistics::Array2D.new(data).descriptive_statistics 54 | # Result: 55 | # 56 | # [{:min=>0.1477, 57 | # :max=>0.6269, 58 | # :mean=>0.347575, 59 | # :median=>0.30785, 60 | # :q1=>0.214975, 61 | # :q3=>0.44045, 62 | # :standard_deviation=>0.18100761551658537}, 63 | # {:min=>0.1055, 64 | # :max=>0.8, 65 | # :mean=>0.38217500000000004, 66 | # :median=>0.3116, 67 | # :q1=>0.1781, 68 | # :q3=>0.515675, 69 | # :standard_deviation=>0.26691825878909076}, 70 | # ..., 71 | # {:min=>0.3918, 72 | # :max=>0.8546, 73 | # :mean=>0.639025, 74 | # :median=>0.6548499999999999, 75 | # :q1=>0.536025, 76 | # :q3=>0.75785, 77 | # :standard_deviation=>0.1718318709523935}] 78 | ``` 79 | 80 | ## Benchmarks 81 | 82 | Some alternatives compared are: 83 | - [descriptive_statistics](https://github.com/thirtysixthspan/descriptive_statistics) 84 | - [ruby-native-statistics](https://github.com/corybuecker/ruby-native-statistics) 85 | - [Numo::NArray](https://github.com/ruby-numo/numo-narray) 86 | - Hand-written ruby (using the same algorithm implemented in C++ in this gem) 87 | 88 | You can reivew the benchmark implementations at `benchmark/benchmark.rb` and run the 89 | benchmark with `rake benchmark`. 90 | 91 | Results: 92 | ``` 93 | Comparing calculated statistics with 10 values for 8 variables... 94 | Test passed, results are equal to 6 decimal places! 95 | 96 | Benchmarking with 100,000 values for 12 variables... 97 | Warming up -------------------------------------- 98 | descriptive_statistics 1.000 i/100ms 99 | Custom ruby 1.000 i/100ms 100 | narray 1.000 i/100ms 101 | ruby_native_statistics 1.000 i/100ms 102 | FastStatistics 3.000 i/100ms 103 | Calculating ------------------------------------- 104 | descriptive_statistics 0.473 (± 0.0%) i/s - 3.000 in 6.354555s 105 | Custom ruby 2.518 (± 0.0%) i/s - 13.000 in 5.169084s 106 | narray 4.231 (± 0.0%) i/s - 22.000 in 5.210299s 107 | ruby_native_statistics 5.962 (± 0.0%) i/s - 30.000 in 5.041869s 108 | FastStatistics 28.417 (±10.6%) i/s - 141.000 in 5.012229s 109 | 110 | Comparison: 111 | FastStatistics: 28.4 i/s 112 | ruby_native_statistics: 6.0 i/s - 4.77x (± 0.00) slower 113 | narray: 4.2 i/s - 6.72x (± 0.00) slower 114 | Custom ruby: 2.5 i/s - 11.29x (± 0.00) slower 115 | descriptive_statistics: 0.5 i/s - 60.09x (± 0.00) slower 116 | ``` 117 | 118 | ## Background & Implementation 119 | 120 | The inspiration for this gem was a use-case in an analytics ruby application, 121 | where we frequently had to compute descriptive statistics for fairly large 122 | multivariate datasets. Calculations in ruby were not fast enough, so I 123 | first explored performing the computations natively in [this 124 | repository](https://github.com/Martin-Nyaga/ruby-ffi-simd). The results were 125 | promising, so I decided to package it as a ruby gem. 126 | 127 | I've now ran this in production for some time, and I'm quite happy with it. Feel 128 | free to let me know in [this discussion 129 | thread](https://github.com/Martin-Nyaga/fast_statistics/discussions/1) if you 130 | use it, or open an issue if you run into any problems. 131 | 132 | ### How is the performance achieved? 133 | The following factors combined help this gem achieve high performance compared 134 | to available native alternatives and hand-written computations in ruby: 135 | 136 | - It is written in C++ and so can leverage the speed of native execution. 137 | - It minimises the number of operations by calculating the statistics in as few 138 | operations as possible (1 sort + 2 loops). Most native alternatives don't 139 | provide a built in way to get all these statistics at once. Instead, they only 140 | provide APIs where you make single calls for individual statistics. Through 141 | such an API, building this set of summary statistics typically ends up looping 142 | through the data more times than is necessary. 143 | - This gem uses explicit 128-bit-wide SIMD intrinsics (on platforms where they 144 | are available) to parallelize computations for 2 variables at the same time 145 | where possible, giving an additional speed advantage while still being single 146 | threaded. 147 | 148 | ### Limitations of the current implementation 149 | The speed gains notwithstanding, there are some limitations in the current implementation: 150 | - The variables in the 2D array must all have the same number of data points 151 | (inner arrays must have the same length) and contain only numbers (i.e. no 152 | `nil` awareness is present). 153 | - There is currently no API to calculate single statistics (although this may be 154 | made available in the future). 155 | 156 | ## Contributing 157 | 158 | Bug reports and pull requests are welcome on GitHub at 159 | https://github.com/Martin-Nyaga/fast_statistics. 160 | 161 | ## License 162 | 163 | The gem is available as open source under the terms of the [MIT 164 | License](https://opensource.org/licenses/MIT). 165 | --------------------------------------------------------------------------------