├── .gitignore ├── 1br.Rproj ├── 1brow.R ├── README.md ├── bbc_plot.R ├── benchmark_fn.R ├── dplyr_and_collapse.R ├── dt.R ├── duckdb_and_arrow.R ├── generate_data.R ├── polars.R └── results.png /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | /data 6 | *.csv -------------------------------------------------------------------------------- /1br.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | -------------------------------------------------------------------------------- /1brow.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(ggplot2) 3 | library(forcats) 4 | library(stringr) 5 | library(readr) 6 | library(here) 7 | 8 | 9 | # import custom ggplot theme (bbc plot) 10 | ## https://bbc.github.io/rcookbook/ 11 | dt_fp <- here::here("dt.R") 12 | 13 | # create file paths to script locations 14 | dplyr_and_collapse_fp <- here::here("dplyr_and_collapse.R") 15 | 16 | dudckdb_and_arrow_fp <- here::here("duckdb_and_arrow.R") 17 | 18 | polars_fp <- here::here("polars.R") 19 | 20 | #load bbcplot 21 | 22 | source("bbc_plot.R") 23 | 24 | # source scripts from terminal to avoid changes to the environment 25 | 26 | system2(dt_fp) 27 | Sys.sleep(2) 28 | system2(dplyr_and_collapse_fp) 29 | Sys.sleep(2) 30 | system2(dudckdb_and_arrow_fp) 31 | Sys.sleep(2) 32 | system2(polars_fp) 33 | 34 | #import results 35 | 36 | files <- list.files(pattern = "_bmarks.csv") 37 | 38 | res <- readr::read_csv(files) 39 | 40 | ## graph results 41 | 42 | res |> 43 | mutate( 44 | time_in_seconds=time/1e9 # convert from nano seconds to seconds 45 | ) |> 46 | ggplot(aes(y=fct_reorder(str_to_upper(expr),time_in_seconds,mean),x=time_in_seconds))+ 47 | geom_boxplot()+ 48 | scale_x_continuous(labels = scales::label_comma(),n.breaks = 10)+ 49 | labs( 50 | x="Seconds" 51 | ,y="Packages" 52 | ,title="Analysis times of 1 billion rows of data" 53 | ,subtitle="Computation (seconds) by popular packages" 54 | ) + 55 | bbc_plot() 56 | 57 | ## save plot png 58 | 59 | ggplot2::ggsave("results.png") 60 | 61 | 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | ![1br_results](https://github.com/alejandrohagan/1br/assets/87534364/e4acbea1-9894-449f-bfb7-6e522f24cc55) 3 | 4 | **note: the data.table resutls shown above are not being reproduced by others even when they run 1 billion rows -- this is under investigation to understand why** 5 | 6 | # Hi! 7 | 8 | - This is the repo inspired by [Gunnar Morlng's](https://www.morling.dev/blog/one-billion-row-challenge/) 1 billion row challenge to see which functions / libraries are quickest in summarizing the mean, min and max of a 1 billion rows of record 9 | 10 | - Actually, [Gunnar's](https://twitter.com/gunnarmorling/) efforts are more robust as its a challenge to the Java community to optimize their code to reduce the processing speed whereas I'm just taking available packages and comparing them 11 | 12 | - I tried to download the orignal Java repo but since I know absolutely nothing about Java this predictiblity failed 13 | - So this isn't the same dataset as the original challenge but it is billion rows of record which I think is the important part of the challenge 14 | 15 | - I used the micro benchmark library as the benchmarking library 16 | - Please feel free to fork or copy the repo and reproduce the findings 17 | 18 | - A few notes: 19 | 20 | - The underlying file is large (~20GB) so that isn't in the repo, however the `generate_data.R` script will produce this for you from your command line (see instructions) 21 | - I ran the analysis in a [Saturn Cloud](https://saturncloud.io/) cluster, my computer struggled with benchmarking billion rows of data multiple times (you can specify how many rows of data you want generated if you want to run it locally) 22 | - Cluster details are: A10G-G54XLarge - 16 cores - 64 GB RAM - 1 GPU - 40Gi Disk 23 | - I'm fairly new to tidypolars and data.table so if you see any issues with the script that I wrote please tell me! 24 | - The general format is that each function test is aggregate by their data workflow eg. since dplyr and collapse use regular data.frames as their input they are kept together and since duckdb and arrow can leverage the same datasource, they are also kept together 25 | - I added `collect()` or whatever the equivalent function is to most of the functions so that they are all returned a tibble format however this may add some variation in the results 26 | 27 | # Instructions 28 | 29 | - Using any flavor of a linux command line run `./generate_data.R` in your command line with the input of rows you want created eg: 30 | ``` 31 | $ ./generate_data.R 1e9 32 | ``` 33 | note: you may need to change the file permission `$ chmod +x generate_data.R` 34 | 35 | - This will create a new folder called "data" which will contain the measurement.txt file 36 | - From there, you can either run each test script individually (eg. dt.R) or you can run them all together via the 1br.R which will source all the scripts 37 | - The format of each test script is largely the same 38 | 39 | - load libraries 40 | - read data (using data.table::fread) 41 | - convert data appropriate format (if necessary) 42 | - create functions that summarize mean, min, max by the respective libraries 43 | - benchmark results 10 times 44 | - save results in csv file 45 | 46 | - Additionally there is a function to covert the ggplot theme to the [bbc plot theme](https://bbc.github.io/rcookbook/) 47 | - There were alot of questions about data.table's benchmark so I'm trying to build an alternative benchmark function using the tictoc package but its wip (`benchmark_fn.R`) 48 | 49 | - If you see any issues or have suggestions of improvements, please let me know! 50 | -------------------------------------------------------------------------------- /bbc_plot.R: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | bbc_plot <- function(){ 5 | 6 | font <- "Helvetica" 7 | 8 | ggplot2::theme( 9 | plot.title = ggplot2::element_text( 10 | family = font 11 | ,size = 28 12 | ,face = "bold" 13 | ,color = "#222222") 14 | ,plot.subtitle = ggplot2::element_text( 15 | family = font 16 | ,size = 22 17 | ,margin = ggplot2::margin(9, 0, 9, 0)) 18 | ,plot.caption = ggplot2::element_blank() 19 | ,legend.position = "top" 20 | ,legend.text.align = 0 21 | ,legend.background = ggplot2::element_blank() 22 | ,legend.title = ggplot2::element_blank() 23 | ,legend.key = ggplot2::element_blank() 24 | ,legend.text = ggplot2::element_text( 25 | family = font 26 | , size = 18 27 | ,color = "#222222") 28 | , axis.title = ggplot2::element_blank() 29 | ,axis.text = ggplot2::element_text( 30 | family = font 31 | ,size = 18 32 | ,color = "#222222") 33 | ,axis.text.x = ggplot2::element_text( 34 | margin = ggplot2::margin(5,b = 10)) 35 | ,axis.ticks = ggplot2::element_blank() 36 | ,axis.line = ggplot2::element_blank() 37 | ,panel.grid.minor = ggplot2::element_blank() 38 | ,panel.grid.major.y = ggplot2::element_line(color = "#cbcbcb") 39 | ,panel.grid.major.x = ggplot2::element_blank() 40 | ,panel.background = ggplot2::element_blank() 41 | ,strip.background = ggplot2::element_rect(fill = "white") 42 | ,strip.text = ggplot2::element_text(size = 22, hjust = 0)) 43 | } -------------------------------------------------------------------------------- /benchmark_fn.R: -------------------------------------------------------------------------------- 1 | library(tictoc) 2 | 3 | benchmark <- function(test_fn){ 4 | tic(log=TRUE,quiet = TRUE) 5 | test_fn 6 | toc(log = TRUE,quiet=TRUE) 7 | } -------------------------------------------------------------------------------- /dplyr_and_collapse.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | library(dplyr) 4 | library(data.table) 5 | library(microbenchmark) 6 | library(collapse) 7 | library(here) 8 | 9 | # free up memory to help with memory mgmt 10 | rm(list=ls()) 11 | 12 | # load data and convert to tbl 13 | 14 | fp <- here::here("measurements.csv") 15 | 16 | measurement_tbl <- data.table::fread(fp) 17 | 18 | # write collapse and dplyr functions 19 | 20 | collapse <- function() { 21 | out <- measurement_tbl |> 22 | fgroup_by(state) |> 23 | fsummarise( 24 | min=fmin(measurement) 25 | ,max=fmax(measurement) 26 | ,mean=fmean(measurement) 27 | ) |> collapse::fungroup() 28 | return(out) 29 | } 30 | 31 | dplyr <- function(){ 32 | out <- measurement_tbl |> 33 | group_by(state) |> 34 | summarise( 35 | min=min(measurement) 36 | ,max=max(measurement) 37 | ,mean=mean(measurement) 38 | ) |> ungroup() 39 | return(out) 40 | } 41 | 42 | 43 | #micro benchmarks 44 | 45 | bmarks <- microbenchmark( 46 | 47 | dplyr=dplyr() 48 | ,collapse=collapse() 49 | ,times = 10 50 | ,unit = "s" 51 | ) 52 | 53 | 54 | write.csv(bmarks,"dplyr_and_collapse_bmarks.csv") 55 | -------------------------------------------------------------------------------- /dt.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | # load libraries 4 | library(data.table) 5 | library(microbenchmark) 6 | library(here) 7 | 8 | # clear name space to help with memory management 9 | rm(list=ls()) 10 | 11 | # load data in DT format 12 | fp <- here::here("measurements.csv") 13 | 14 | measurement_dt <- data.table::fread(fp) 15 | 16 | 17 | # create function to simulate results 18 | 19 | dt <- function(){ 20 | 21 | out <- measurement_dt[ ,.(mean=mean(measurement),min=min(measurement),max=max(measurement)),by=state] 22 | 23 | return(out) 24 | 25 | } 26 | 27 | 28 | #micro benchmarks 29 | 30 | bmarks <- microbenchmark::microbenchmark( 31 | 32 | dt=dt() 33 | ,times = 10 34 | , unit = "s") 35 | 36 | # save results 37 | 38 | write.csv(bmarks,"dt_bmarks.csv") 39 | 40 | -------------------------------------------------------------------------------- /duckdb_and_arrow.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | #library 4 | library(DBI) 5 | library(duckdb) 6 | library(dplyr) 7 | library(microbenchmark) 8 | library(data.table) 9 | library(arrow) 10 | library(here) 11 | 12 | #load data and create duckdb connection 13 | 14 | fp <- here::here("measurements.csv") 15 | 16 | measurement_tbl <- data.table::fread(fp) 17 | 18 | con <- DBI::dbConnect(duckdb::duckdb()) 19 | 20 | duckdb::duckdb_register(con,name = "measurement_db",measurement_tbl) 21 | 22 | measurement_db <- dplyr::tbl(con,"measurement_db") 23 | 24 | measurements_ar <- measurement_db |> arrow::to_arrow() 25 | 26 | 27 | # create function to simulate results 28 | 29 | duckdb <- function(){ 30 | out <- measurement_db |> 31 | group_by(state) |> 32 | summarize( 33 | min=min(measurement,na.rm=TRUE) 34 | ,max=max(measurement,na.rm=TRUE) 35 | ,mean=mean(measurement,na.rm=TRUE) 36 | ) |> ungroup() |> 37 | collect() 38 | 39 | return(out) 40 | } 41 | 42 | 43 | # arrow <- function(){ 44 | # 45 | # out <- measurements_ar |> 46 | # group_by(state) |> 47 | # summarize( 48 | # min=min(measurement,na.rm=TRUE) 49 | # ,max=max(measurement,na.rm=TRUE) 50 | # ,mean=mean(measurement,na.rm=TRUE) 51 | # ) |> 52 | # ungroup() |> 53 | # collect() 54 | # 55 | # return(out) 56 | # 57 | # } 58 | 59 | 60 | 61 | bmarks <- microbenchmark( 62 | 63 | duckdb=duckdb() 64 | # ,arrow=arrow() 65 | ,times = 10 66 | , unit = "s" 67 | ) 68 | 69 | 70 | write.csv(bmarks,"duckdb_and_arrow_bmarks.csv") 71 | 72 | 73 | -------------------------------------------------------------------------------- /generate_data.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | library(data.table) 4 | library(datasets) 5 | 6 | print("creating dataset") 7 | 8 | input <- base::commandArgs(trailingOnly = TRUE) 9 | 10 | input <- base::as.numeric(input) 11 | 12 | dataset_message <- paste0("dataset will be ",input," rows") 13 | 14 | print(dataset_message) 15 | 16 | set.seed(2024) 17 | 18 | measurement_tbl <-data.frame( 19 | measurement = stats::rnorm(input) 20 | ,state = base::sample(datasets::state.abb, size = input, replace = TRUE) 21 | ) 22 | 23 | 24 | print("dataset created, beginning saving dataset") 25 | 26 | 27 | file = "measurements.csv" 28 | 29 | data.table::fwrite(measurement_tbl, "measurements.csv") 30 | 31 | 32 | size = structure(file.info(file)$size, class = "object_size") |> format("auto") 33 | 34 | 35 | 36 | message("Finished saving dataset \"", file, "\". Its file size is: ", size, ".") 37 | 38 | -------------------------------------------------------------------------------- /polars.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | # load libraries 4 | 5 | library(tidypolars) 6 | library(dplyr) 7 | library(microbenchmark) 8 | library(data.table) 9 | library(here) 10 | 11 | # clear name space to hlep with memory management 12 | rm(list=ls()) 13 | 14 | # load data and convert to polars tbl 15 | 16 | fp <- here::here("measurements.csv") 17 | 18 | measurement_pl <- data.table::fread(fp) |> 19 | as_polars() 20 | 21 | 22 | 23 | 24 | # create function to simulate results 25 | 26 | polars <- function(){ 27 | 28 | out <- measurement_pl |> 29 | group_by(state) |> 30 | summarise( 31 | min=min(measurement) 32 | ,max=max(measurement) 33 | ,mean=mean(measurement) 34 | ) |> 35 | ungroup() |> 36 | tidypolars::to_r() 37 | return(out) 38 | } 39 | 40 | # benchmark results 41 | bmarks <- microbenchmark( 42 | 43 | polars=polars() 44 | ,times = 10 45 | , unit = "s" 46 | ) 47 | 48 | # save results 49 | 50 | write.csv(bmarks,"polars_bmarks.csv") -------------------------------------------------------------------------------- /results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alejandrohagan/1br/83615d337d6a0684857e7f526b4bf1505bc8f40e/results.png --------------------------------------------------------------------------------