├── .RBuildignore ├── .gitattributes ├── vignettes ├── .gitignore ├── version.compare.md └── version.compare.Rmd ├── LICENSE ├── tests ├── test-all.R └── testthat │ └── test-1-RevoBenchmark.R ├── inst ├── examples │ ├── example_RevoBenchmark.R │ └── example_version.time.R └── doc │ ├── version.compare.R │ └── version.compare.Rmd ├── .gitignore ├── NAMESPACE ├── man ├── urbanekPerformance.Rd ├── plot.RevoBenchmark.Rd ├── findRscript.Rd ├── version.time.Rd ├── RevoMultiBenchmark.Rd └── RevoBenchmark.Rd ├── DESCRIPTION ├── README.md └── R ├── RevoMultibenchmark.R ├── RevoBenchmark-methods.R ├── version.time.R ├── findRscript.R └── RevoBenchmark.R /.RBuildignore: -------------------------------------------------------------------------------- 1 | .gitignore 2 | README.md 3 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.html linguist-vendored 2 | -------------------------------------------------------------------------------- /vignettes/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.R 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | YEAR: 2015 2 | COPYRIGHT HOLDER: Andrie de Vries 3 | -------------------------------------------------------------------------------- /tests/test-all.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | test_check("version.compare") 3 | -------------------------------------------------------------------------------- /inst/examples/example_RevoBenchmark.R: -------------------------------------------------------------------------------- 1 | # Run Urbanek benchmark 2 | # Set low scale.factor to reduce time to run on CRAN 3 | # For actual testing, use the default, i.e. scale.factor = 1 4 | RevoBenchmark(threads = c(4, 8), scale.factor = 0.1) 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # History files 2 | .Rhistory 3 | 4 | # Example code in package build process 5 | *-Ex.R 6 | 7 | # R data files from past sessions 8 | .Rdata 9 | 10 | # RStudio files 11 | .Rproj.user/ 12 | 13 | # Vignette cache 14 | vignettes/*_cache/* 15 | vignettes/*_files/* 16 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | S3method(plot,RevoBenchmark) 4 | export(RevoBenchmark) 5 | export(RevoMultiBenchmark) 6 | export(findRscript) 7 | export(urbanekPerformance) 8 | export(version.time) 9 | import(ggplot2) 10 | importFrom(MASS,lda) 11 | importFrom(reshape2,melt) 12 | -------------------------------------------------------------------------------- /man/urbanekPerformance.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/RevoBenchmark-methods.R 3 | \name{urbanekPerformance} 4 | \alias{urbanekPerformance} 5 | \title{Computes urbanek performance improvement.} 6 | \usage{ 7 | urbanekPerformance(x) 8 | } 9 | \arguments{ 10 | \item{x}{RevoBenchmark object} 11 | } 12 | \description{ 13 | Computes urbanek performance improvement. 14 | } 15 | 16 | -------------------------------------------------------------------------------- /man/plot.RevoBenchmark.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/RevoBenchmark-methods.R 3 | \name{plot.RevoBenchmark} 4 | \alias{plot.RevoBenchmark} 5 | \title{Create plot of RevoBenchmark results.} 6 | \usage{ 7 | \method{plot}{RevoBenchmark}(x, theme_size = 16, 8 | main = "Elapsed time in seconds", ...) 9 | } 10 | \arguments{ 11 | \item{x}{RevoBenchmark object} 12 | 13 | \item{theme_size}{Passed to \code{\link[ggplot2]{theme_bw}}.} 14 | 15 | \item{...}{Not used} 16 | } 17 | \description{ 18 | Create plot of RevoBenchmark results. 19 | } 20 | 21 | -------------------------------------------------------------------------------- /inst/examples/example_version.time.R: -------------------------------------------------------------------------------- 1 | # Find installed versions of Rscript 2 | 3 | rversion <- paste(R.version$major, R.version$minor, sep = ".") 4 | rscript <- findRscript(version = rversion) 5 | 6 | # Configure which installed version to use 7 | 8 | # Compute vector mean in different R installations 9 | 10 | version.time(rscript, { 11 | set.seed(1) 12 | foo <- rnorm(1e4) 13 | mean(foo) 14 | }) 15 | 16 | 17 | # Compute matrix cross product in different R installations 18 | 19 | version.time(rscript, { 20 | set.seed(42) 21 | m <- matrix(runif(100), nrow=10) 22 | crossprod(m) 23 | }) 24 | 25 | -------------------------------------------------------------------------------- /vignettes/version.compare.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Run MKL benchmarks" 3 | author: "Andrie de Vries" 4 | date: "2016-02-11" 5 | output: rmarkdown::html_vignette 6 | vignette: > 7 | %\VignetteIndexEntry{Run MKL benchmarks} 8 | %\VignetteEngine{knitr::rmarkdown} 9 | %\usepackage[utf8]{inputenc} 10 | --- 11 | 12 | # Running the tests 13 | 14 | 15 | 16 | The package allows you to run benchmark tests on several different installations of R on the same machine. 17 | 18 | This vignette scales the test sets down to 10% of standard size, to speed up the process for sake of illustration. For a full-scale test, adjust the tests by setting `scale.factor = 1`. 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: version.compare 2 | Title: Compares the Results of Running Code in Different Installed Versions of R 3 | Version: 0.1.2 4 | Authors@R: "Andrie de Vries [aut, cre]" 5 | Date: 2017-03-22 6 | Description: Allows you to run R code in different versions of R and compare 7 | results. In addition, provides an easy mechanism to run a modified version of 8 | the Urbanek benchmark tests. 9 | License: MIT + file LICENSE 10 | Copyright: Microsoft 11 | URL: http://www.github.com/andrie/version.compare 12 | BugReports: http://www.github.com/andrie/version.compare/issues 13 | LazyData: true 14 | Depends: 15 | R (>= 3.0.0), 16 | Imports: 17 | MASS, 18 | ggplot2, 19 | reshape2 20 | Suggests: 21 | testthat, 22 | knitr 23 | VignetteBuilder: knitr 24 | RoxygenNote: 5.0.1 25 | -------------------------------------------------------------------------------- /tests/testthat/test-1-RevoBenchmark.R: -------------------------------------------------------------------------------- 1 | if(interactive()) library(testthat) 2 | context("RevoBenchmark") 3 | 4 | describe("Activates MKL if it exists",{ 5 | test_that("RevoBencmark takes single threads argument",{ 6 | it("tests for correct class", { 7 | p1 <- RevoBenchmark(threads = 1, scale.factor = 0.05) 8 | p2 <- RevoBenchmark(threads = 2, scale.factor = 0.05) 9 | 10 | expect_is(p1, c("RevoBenchmark", "matrix")) 11 | expect_is(p1, c("RevoBenchmark", "matrix")) 12 | expect_is(plot(p1), "ggplot") 13 | expect_is(plot(p2), "ggplot") 14 | 15 | }) 16 | }) 17 | test_that("RevoBencmark takes multiple threads arguments",{ 18 | it("tests for correct class", { 19 | p <- RevoBenchmark(threads = c(1, 2), scale.factor = 0.05) 20 | 21 | expect_is(p, c("RevoBenchmark", "matrix")) 22 | expect_is(plot(p), "ggplot") 23 | 24 | }) 25 | }) 26 | }) 27 | 28 | -------------------------------------------------------------------------------- /man/findRscript.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/findRscript.R 3 | \name{findRscript} 4 | \alias{findRscript} 5 | \title{Returns list of R installation paths.} 6 | \usage{ 7 | findRscript(path, executable = "(Rscript$|Rscript.exe)", pattern, version) 8 | } 9 | \arguments{ 10 | \item{path}{Character vector of search paths. Passed to \code{\link{list.files}}. If not specified (the default) then the search path contains typical installation paths for different operating systems.} 11 | 12 | \item{executable}{Regular expression that defines the name of the Rscript executable. Defaults to a regular expression that finds \code{Rscript} on linux machines and \code{Rscript.exe} on Windows} 13 | 14 | \item{pattern}{regular expression. Passed to \code{\link{list.files}}} 15 | 16 | \item{version}{Additional regular expression, e.g. "3.2.1" to limit the results list to a specific version of R} 17 | } 18 | \description{ 19 | Given a vector of search paths, returns all instances of R installations. 20 | } 21 | \details{ 22 | This function is a wrapper around \code{list.files} setting the search pattern to \code{Rscript.exe}. 23 | } 24 | 25 | -------------------------------------------------------------------------------- /man/version.time.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/version.time.R 3 | \name{version.time} 4 | \alias{version.time} 5 | \title{Run script in different versions of R and capture system time.} 6 | \usage{ 7 | version.time(rVersions, expr, file) 8 | } 9 | \arguments{ 10 | \item{rVersions}{Character vector containing path to Rscript in different installations of R} 11 | 12 | \item{expr}{Expression to evaluate} 13 | 14 | \item{file}{If specified, evaluates the code in the file, rather than \code{expr}} 15 | } 16 | \description{ 17 | Run script in different versions of R and capture system time. 18 | } 19 | \examples{ 20 | # Find installed versions of Rscript 21 | 22 | rversion <- paste(R.version$major, R.version$minor, sep = ".") 23 | rscript <- findRscript(version = rversion) 24 | 25 | # Configure which installed version to use 26 | 27 | # Compute vector mean in different R installations 28 | 29 | version.time(rscript, { 30 | set.seed(1) 31 | foo <- rnorm(1e4) 32 | mean(foo) 33 | }) 34 | 35 | 36 | # Compute matrix cross product in different R installations 37 | 38 | version.time(rscript, { 39 | set.seed(42) 40 | m <- matrix(runif(100), nrow=10) 41 | crossprod(m) 42 | }) 43 | 44 | } 45 | 46 | -------------------------------------------------------------------------------- /man/RevoMultiBenchmark.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/RevoMultibenchmark.R 3 | \name{RevoMultiBenchmark} 4 | \alias{RevoMultiBenchmark} 5 | \title{Runs Urbanek tests on multiple versions of R and compares results.} 6 | \usage{ 7 | RevoMultiBenchmark(rVersions, threads = 4, scale.factor = 1) 8 | } 9 | \arguments{ 10 | \item{rVersions}{Character vector containing path to Rscript in different installations of R} 11 | 12 | \item{threads}{Number of Intel MKL threads to use if availalbe. Tests for the presence of the packages \code{RevoUtilsMath} or \code{RevoBase} and sets the number of threads using \code{RevoUtilsMath::setMKLthreads}} 13 | 14 | \item{scale.factor}{A numeric value that scales the size of the data up or down. The default value of 1 has data sizes that yields a runtime of ~2 seconds per test on an 8-core machine with the MKL available. For quick and easy testing, reduce the \code{scale.factor} to less than 1. (The primary use case for low \code{scale.factor} is to reduce the unit testing time when testing the package itself on CRAN.) To scale out the test data, use \code{scale.factor} of greater than 1.} 15 | } 16 | \description{ 17 | Runs Urbanek tests on multiple versions of R and compares results. 18 | } 19 | 20 | -------------------------------------------------------------------------------- /man/RevoBenchmark.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/RevoBenchmark.R 3 | \name{RevoBenchmark} 4 | \alias{RevoBenchmark} 5 | \title{Returns the system time for a modified version of the Urbanek benchmarks.} 6 | \usage{ 7 | RevoBenchmark(threads = 4, show.message = TRUE, scale.factor = 1) 8 | } 9 | \arguments{ 10 | \item{threads}{Number of Intel MKL threads to use if availalbe. Tests for the presence of the packages \code{RevoUtilsMath} or \code{RevoBase} and sets the number of threads using \code{RevoUtilsMath::setMKLthreads}} 11 | 12 | \item{show.message}{If TRUE, shows interim results as console messages} 13 | 14 | \item{scale.factor}{A numeric value that scales the size of the data up or down. The default value of 1 has data sizes that yields a runtime of ~2 seconds per test on an 8-core machine with the MKL available. For quick and easy testing, reduce the \code{scale.factor} to less than 1. (The primary use case for low \code{scale.factor} is to reduce the unit testing time when testing the package itself on CRAN.) To scale out the test data, use \code{scale.factor} of greater than 1.} 15 | } 16 | \description{ 17 | Returns the system time for a modified version of the Urbanek benchmarks. 18 | } 19 | \seealso{ 20 | plot.RevoBenchmark 21 | } 22 | 23 | -------------------------------------------------------------------------------- /inst/doc/version.compare.R: -------------------------------------------------------------------------------- 1 | ## ------------------------------------------------------------------------ 2 | library("version.compare") 3 | library("knitr") 4 | scale.factor <- 1.0 5 | 6 | ## ----benchmark, echo=TRUE------------------------------------------------ 7 | 8 | r <- switch(Sys.info()[["sysname"]], 9 | Linux = { 10 | rscript <- findRscript() 11 | 12 | rv <- version.time(rscript, { 13 | as.character(getRversion()) 14 | }) 15 | idx <- which(unlist(rv$results) == "3.4.1") 16 | rscript[idx] 17 | 18 | }, 19 | Windows = findRscript(version = "3.4.1.*x64" 20 | ) 21 | ) 22 | test.results <- RevoMultiBenchmark(rVersions = r, 23 | threads = c(1, 4, 8), 24 | scale.factor = scale.factor) 25 | 26 | ## ----compare-results, eval = FALSE--------------------------------------- 27 | # kable(test.results) 28 | # plot(test.results, theme_size = 8, main = "Elapsed time") 29 | 30 | ## ----performance-speed-up, eval = FALSE---------------------------------- 31 | # kable(urbanekPerformance(test.results), digits = 2) 32 | # plot(urbanekPerformance(test.results), theme_size = 8, main = "Relative Performance") 33 | 34 | ## ----compare-results, echo = FALSE, fig.width=6, fig.height=3, out.width="800px", dpi=1200, keep=TRUE---- 35 | kable(test.results) 36 | plot(test.results, theme_size = 8, main = "Elapsed time") 37 | 38 | ## ----performance-speed-up, echo = FALSE, fig.width=6, fig.height=3, out.width="800px", dpi=1200---- 39 | kable(urbanekPerformance(test.results), digits = 2) 40 | plot(urbanekPerformance(test.results), theme_size = 8, main = "Relative Performance") 41 | 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Compare the results of R code running in different installed versions of R 2 | 3 | 4 | ## Objective: 5 | 6 | * To compare results (and execution speed) of a script running in different installed versions of R 7 | * This is done by using RScript and diverting output to temp files 8 | 9 | 10 | ## Exported functions 11 | 12 | * `findRscript()` 13 | - Finds installed versions of R on a machine, by searching for `Rscript` in typical installation folders 14 | 15 | * `version.time()` 16 | - Similar to `system.time()` in base R, takes an expression as input and runs this expression in multiple installations of R on the same machine 17 | - Returns results as well as `system.time()` for each installed version. 18 | 19 | 20 | ## Example 21 | 22 | This example runs a simple script in two different installations of R. 23 | 24 | ```r 25 | # Find installed versions of Rscript 26 | 27 | rscript <- findRscript() 28 | 29 | 30 | # Configure which installed version to use 31 | 32 | rscript <- switch( 33 | Sys.info()[["sysname"]], 34 | Windows = c( 35 | "c:/program files/RRO/R-3.1.1/bin/x64/Rscript.exe", 36 | "c:/program files/R/R-3.1.1/bin/x64/Rscript.exe" 37 | ), 38 | Linux = c( 39 | "/usr/lib64/RRO-8.0/R-3.1.1/lib/R/bin/Rscript", 40 | "/usr/lib/R/bin/Rscript" 41 | ) 42 | ) 43 | 44 | # Compute vector mean in different R installations 45 | 46 | version.time({ 47 | foo <- rnorm(1e6) 48 | mean(foo) 49 | } , rscript) 50 | 51 | 52 | # Compute matrix cross product in different R installations 53 | 54 | version.time({ 55 | m <- matrix(runif(100), nrow=10) 56 | crossprod(m) 57 | } , rscript) 58 | ``` 59 | 60 | # View the vignette 61 | 62 | You can see the package vignette at https://htmlpreview.github.io/?https://github.com/andrie/version.compare/blob/master/inst/doc/version.compare.html 63 | -------------------------------------------------------------------------------- /R/RevoMultibenchmark.R: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015, Andrie de Vries 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining 4 | # a copy of this software and associated documentation files (the 5 | # "Software"), to deal in the Software without restriction, including 6 | # without limitation the rights to use, copy, modify, merge, publish, 7 | # distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so, subject to 9 | # the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be 12 | # included in all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | 23 | #' Runs Urbanek tests on multiple versions of R and compares results. 24 | #' 25 | #' @inheritParams version.time 26 | #' @inheritParams RevoBenchmark 27 | #' 28 | #' @export 29 | RevoMultiBenchmark <- function(rVersions, threads = 4, scale.factor = 1){ 30 | 31 | paste.deparse <- function(x, collapse = "\n") paste(deparse(x), collapse = collapse) 32 | 33 | tf <- tempfile(pattern = "urbanek-", fileext = ".r") 34 | code <- paste("urbanek2.5 <- ", paste.deparse(RevoBenchmark)) 35 | fn <- sprintf("urbanek2.5(threads = %s, scale.factor = %s)", paste.deparse(threads, collapse = " "), scale.factor) 36 | code <- paste(code, fn, sep = "\n") 37 | write(code, file = tf) 38 | on.exit(unlink(tf)) 39 | z <- version.time(rVersions, file = tf)[["results"]] 40 | urbanekCombine(z) 41 | } 42 | -------------------------------------------------------------------------------- /inst/doc/version.compare.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Run MKL benchmarks" 3 | author: "Andrie de Vries" 4 | date: "`r Sys.Date()`" 5 | output: rmarkdown::html_vignette 6 | vignette: > 7 | %\VignetteIndexEntry{Run MKL benchmarks} 8 | %\VignetteEngine{knitr::rmarkdown} 9 | %\usepackage[utf8]{inputenc} 10 | --- 11 | 12 | # Running the tests 13 | 14 | The package allows you to run benchmark tests on several different installations of R on the same machine. 15 | 16 | The benchmarks are an adaptation of [Simon Urbanek's benchmarks](http://r.research.att.com/benchmarks/R-benchmark-25.R), and have been included in the function `RevoMultiBenchmark()` in the `version.compare` package. 17 | 18 | ```{r} 19 | library("version.compare") 20 | library("knitr") 21 | scale.factor <- 1.0 22 | ``` 23 | 24 | This vignette scales the test sets down to `r sprintf("%1.0f%%", 100*scale.factor)` of standard size, to speed up the process for sake of illustration. For a full-scale test, adjust the tests by setting `scale.factor <- 1`. 25 | 26 | ```{r benchmark, echo=TRUE} 27 | 28 | r <- switch(Sys.info()[["sysname"]], 29 | Linux = { 30 | rscript <- findRscript() 31 | 32 | rv <- version.time(rscript, { 33 | as.character(getRversion()) 34 | }) 35 | idx <- which(unlist(rv$results) == "3.4.1") 36 | rscript[idx] 37 | 38 | }, 39 | Windows = findRscript(version = "3.4.1.*x64" 40 | ) 41 | ) 42 | test.results <- RevoMultiBenchmark(rVersions = r, 43 | threads = c(1, 4, 8), 44 | scale.factor = scale.factor) 45 | ``` 46 | 47 | ```{r compare-results, eval = FALSE} 48 | kable(test.results) 49 | plot(test.results, theme_size = 8, main = "Elapsed time") 50 | ``` 51 | 52 | ```{r performance-speed-up, eval = FALSE} 53 | kable(urbanekPerformance(test.results), digits = 2) 54 | plot(urbanekPerformance(test.results), theme_size = 8, main = "Relative Performance") 55 | ``` 56 | 57 | # Compare results 58 | 59 | ## Elapsed time (seconds) 60 | 61 | The package captures the total elapsed time for each of the benchmark tests. This makes it easy to compare the total elapsed time (in seconds). 62 | 63 | 64 | ```{r compare-results, echo = FALSE, fig.width=6, fig.height=3, out.width="800px", dpi=1200, keep=TRUE} 65 | ``` 66 | 67 | ## Relative performance 68 | 69 | You may also want to compare relative performance. To do this, divide each column by the longest-running test. This means scaling is relative to the version of R that has the longest elapsed time. 70 | 71 | 72 | ```{r performance-speed-up, echo = FALSE, fig.width=6, fig.height=3, out.width="800px", dpi=1200} 73 | ``` 74 | -------------------------------------------------------------------------------- /vignettes/version.compare.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Run MKL benchmarks" 3 | author: "Andrie de Vries" 4 | date: "`r Sys.Date()`" 5 | output: rmarkdown::html_vignette 6 | vignette: > 7 | %\VignetteIndexEntry{Run MKL benchmarks} 8 | %\VignetteEngine{knitr::rmarkdown} 9 | %\usepackage[utf8]{inputenc} 10 | --- 11 | 12 | # Running the tests 13 | 14 | The package allows you to run benchmark tests on several different installations of R on the same machine. 15 | 16 | The benchmarks are an adaptation of [Simon Urbanek's benchmarks](http://r.research.att.com/benchmarks/R-benchmark-25.R), and have been included in the function `RevoMultiBenchmark()` in the `version.compare` package. 17 | 18 | ```{r} 19 | library("version.compare") 20 | library("knitr") 21 | scale.factor <- 1.0 22 | ``` 23 | 24 | This vignette scales the test sets down to `r sprintf("%1.0f%%", 100*scale.factor)` of standard size, to speed up the process for sake of illustration. For a full-scale test, adjust the tests by setting `scale.factor <- 1`. 25 | 26 | ```{r benchmark, echo=TRUE} 27 | 28 | r <- switch(Sys.info()[["sysname"]], 29 | Linux = { 30 | rscript <- findRscript() 31 | 32 | rv <- version.time(rscript, { 33 | as.character(getRversion()) 34 | }) 35 | idx <- which(unlist(rv$results) == "3.4.1") 36 | rscript[idx] 37 | 38 | }, 39 | Windows = findRscript(version = "3.4.1.*x64" 40 | ) 41 | ) 42 | test.results <- RevoMultiBenchmark(rVersions = r, 43 | threads = c(1, 4, 8), 44 | scale.factor = scale.factor) 45 | ``` 46 | 47 | ```{r compare-results, eval = FALSE} 48 | kable(test.results) 49 | plot(test.results, theme_size = 8, main = "Elapsed time") 50 | ``` 51 | 52 | ```{r performance-speed-up, eval = FALSE} 53 | kable(urbanekPerformance(test.results), digits = 2) 54 | plot(urbanekPerformance(test.results), theme_size = 8, main = "Relative Performance") 55 | ``` 56 | 57 | # Compare results 58 | 59 | ## Elapsed time (seconds) 60 | 61 | The package captures the total elapsed time for each of the benchmark tests. This makes it easy to compare the total elapsed time (in seconds). 62 | 63 | 64 | ```{r compare-results, echo = FALSE, fig.width=6, fig.height=3, out.width="800px", dpi=1200, keep=TRUE} 65 | ``` 66 | 67 | ## Relative performance 68 | 69 | You may also want to compare relative performance. To do this, divide each column by the longest-running test. This means scaling is relative to the version of R that has the longest elapsed time. 70 | 71 | 72 | ```{r performance-speed-up, echo = FALSE, fig.width=6, fig.height=3, out.width="800px", dpi=1200} 73 | ``` 74 | -------------------------------------------------------------------------------- /R/RevoBenchmark-methods.R: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015, Andrie de Vries 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining 4 | # a copy of this software and associated documentation files (the 5 | # "Software"), to deal in the Software without restriction, including 6 | # without limitation the rights to use, copy, modify, merge, publish, 7 | # distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so, subject to 9 | # the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be 12 | # included in all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | 23 | # Combines results from Urbanek tests 24 | urbanekCombine <- function(x){ 25 | z <- do.call(cbind, x) 26 | z <- z[, order(colSums(z), decreasing = TRUE)] 27 | class(z) <- c("RevoBenchmark", "matrix") 28 | z 29 | } 30 | 31 | 32 | to.data.frame <- function(x){ 33 | if(is.data.frame(x)) return(x) 34 | dat <- cbind(test = rownames(x), data.frame(as.matrix(x), stringsAsFactors = FALSE, check.names = FALSE)) 35 | rownames(dat) <- seq_len(nrow(dat)) 36 | dat 37 | } 38 | 39 | #' Computes urbanek performance improvement. 40 | #' 41 | #' @param x RevoBenchmark object 42 | #' @export 43 | urbanekPerformance <- function(x){ 44 | # if(!is.data.frame(x)) x <- urbanekCombine(x) 45 | # 1 / (x / x[, 1]) - 1 46 | 1 / (x / x[, 1]) 47 | 48 | } 49 | 50 | 51 | 52 | 53 | #' Create plot of RevoBenchmark results. 54 | #' 55 | #' @inheritParams urbanekPerformance 56 | #' @param ... Not used 57 | #' @param theme_size Passed to \code{\link[ggplot2]{theme_bw}}. 58 | #' 59 | #' @import ggplot2 60 | #' @importFrom reshape2 melt 61 | #' @export 62 | plot.RevoBenchmark <- function(x, theme_size=16, main = "Elapsed time in seconds", ...){ 63 | dat <- to.data.frame(x) 64 | minmax <- if(ncol(dat) > 2){ 65 | data.frame( 66 | test = dat[, "test"], 67 | min = apply(dat[, -1], 1, min), 68 | max = apply(dat[, -1], 1, max) 69 | ) 70 | } else { 71 | data.frame( 72 | test = dat[, "test"], 73 | min = dat[, -1], 74 | max = dat[, -1] 75 | ) 76 | 77 | } 78 | mdat <- reshape2::melt(dat, id.var = "test") 79 | ggplot2::ggplot(mdat, aes_string(x = "test")) + 80 | geom_linerange(data = minmax, 81 | aes_string(ymin = "min", ymax = "max") 82 | ) + 83 | geom_crossbar(data = mdat, 84 | aes_string(y = "value", ymin = "value", ymax = "value", col = "variable"), 85 | size = 0.3 86 | ) + 87 | geom_point(data = mdat, 88 | aes_string(y = "value", col = "variable"), 89 | stat = "identity", 90 | size = 3 91 | ) + 92 | coord_flip() + 93 | xlab(NULL) + 94 | ylab(NULL) + 95 | ggtitle(main) + 96 | scale_color_discrete("R version") + 97 | theme_bw(theme_size) 98 | } 99 | -------------------------------------------------------------------------------- /R/version.time.R: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015, Andrie de Vries 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining 4 | # a copy of this software and associated documentation files (the 5 | # "Software"), to deal in the Software without restriction, including 6 | # without limitation the rights to use, copy, modify, merge, publish, 7 | # distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so, subject to 9 | # the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be 12 | # included in all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | 23 | #' Run script in different versions of R and capture system time. 24 | #' 25 | #' @param rVersions Character vector containing path to Rscript in different installations of R 26 | #' @param expr Expression to evaluate 27 | #' @param file If specified, evaluates the code in the file, rather than \code{expr} 28 | #' @export 29 | #' 30 | #' @example \inst\examples\example_version.time.R 31 | version.time <- function(rVersions, expr, file){ 32 | match <- match.call() 33 | expr <- match$expr 34 | 35 | # capture system.time of original expression and write results to tempfile 36 | p0 <- "library('methods')" 37 | p1 <- "result <- local({" 38 | p2 <- "}) 39 | tf <- normalizePath(tempfile(fileext = '.rds', tmpdir = dirname(tempdir())), mustWork = FALSE) 40 | saveRDS(result, file = tf) 41 | cat(tf) 42 | " 43 | scriptfile <- tempfile(fileext = ".R") 44 | on.exit(unlink(scriptfile)) 45 | 46 | 47 | 48 | if(!missing("file") && !is.null(file)){ 49 | tmp <- paste(readLines(file), collapse = "\n") 50 | } else { 51 | tmp <- paste(deparse(expr), collapse = "\n") 52 | } 53 | new.expr <- sprintf("%s\n%s\n%s\n%s", p0, p1, tmp, p2) 54 | 55 | # save new expression to file so it can be picked up by clean R session 56 | write(new.expr, file = scriptfile) 57 | 58 | # cat(readLines(scriptfile), sep = "\n") 59 | 60 | 61 | 62 | # run the actual script in each R version 63 | res <- lapply(rVersions, runWithRscript, testscriptPath = scriptfile) 64 | result.tempfiles = sapply(res, function(x)tail(x[["stdout"]], 1)) 65 | mres <- list( 66 | time = lapply(res, function(x)x[["time"]]), 67 | stdout = lapply(res, function(x)head(x[["stdout"]], -1)) 68 | ) 69 | 70 | res <- NULL 71 | 72 | # retrieve results stored in tempfiles 73 | on.exit(unlink(result.tempfiles)) 74 | mres[["results"]] <- lapply(result.tempfiles, readRDS) 75 | 76 | mres 77 | } 78 | 79 | 80 | 81 | runWithRscript <- function(rscriptPath, testscriptPath, message=TRUE){ 82 | cmd <- paste( 83 | shQuote(rscriptPath), 84 | shQuote(testscriptPath) 85 | ) 86 | if(message) message(rscriptPath) 87 | stdout <- NA 88 | time <- system.time({ stdout <- system(cmd, intern = TRUE) }) 89 | list(time = time, stdout = stdout) 90 | } 91 | 92 | 93 | -------------------------------------------------------------------------------- /R/findRscript.R: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015, Andrie de Vries 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining 4 | # a copy of this software and associated documentation files (the 5 | # "Software"), to deal in the Software without restriction, including 6 | # without limitation the rights to use, copy, modify, merge, publish, 7 | # distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so, subject to 9 | # the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be 12 | # included in all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | 23 | # Searches windows registry for installed versions of R 24 | findRinRegistry <- function(){ 25 | readOnce <- function(ptn){ 26 | x <- readRegistry(ptn, maxdepth = 3) 27 | if(length(x)){ 28 | y <- unlist(x, recursive = TRUE) 29 | unname(y[grep(".InstallPath", names(y))]) 30 | } else NULL 31 | } 32 | patterns <- c("SOFTWARE\\Microsoft", "SOFTWARE\\R-core") 33 | unique(unname(unlist(sapply(patterns, readOnce)))) 34 | } 35 | 36 | 37 | 38 | #' Returns list of R installation paths. 39 | #' 40 | #' Given a vector of search paths, returns all instances of R installations. 41 | #' 42 | #' 43 | #' @details 44 | #' This function is a wrapper around \code{list.files} setting the search pattern to \code{Rscript.exe}. 45 | #' 46 | #' @param path Character vector of search paths. Passed to \code{\link{list.files}}. If not specified (the default) then the search path contains typical installation paths for different operating systems. 47 | #' @param executable Regular expression that defines the name of the Rscript executable. Defaults to a regular expression that finds \code{Rscript} on linux machines and \code{Rscript.exe} on Windows 48 | #' @param pattern regular expression. Passed to \code{\link{list.files}} 49 | #' @param version Additional regular expression, e.g. "3.2.1" to limit the results list to a specific version of R 50 | #' 51 | #' @export 52 | findRscript <- function(path, executable = "(Rscript$|Rscript.exe)", pattern, version){ 53 | 54 | foo <- function(path, executable, recursive){ 55 | # message("Now searching ", path) 56 | list.files(path=path, pattern=executable, full.names=TRUE, include.dirs=TRUE, recursive=recursive) 57 | } 58 | 59 | if(missing(path) || is.null(path)){ 60 | path <- switch(Sys.info()[["sysname"]], 61 | Linux = c("/usr/lib64", "/usr/lib"), 62 | # Windows = c("c:/program files", "c:/program files (x86)", "c:/R", "c:/revolution", "c:/rro"), 63 | Windows = sort(unique(dirname(findRinRegistry()))), 64 | c("/Library/Frameworks/") 65 | ) 66 | } 67 | 68 | if(missing(executable) || is.null(executable)){ 69 | executable <- switch(Sys.info()[["sysname"]], 70 | Linux = "Rscript$", 71 | Windows = "Rscript.exe$", 72 | "Rscript$" 73 | ) 74 | } 75 | 76 | ret <- lapply(path, foo, executable=executable, recursive=TRUE) 77 | ret <- unique(unname(unlist(ret))) 78 | if(!missing("version") && !is.null(version)) ret <- ret[grepl(version, ret)] 79 | normalizePath(ret) 80 | } 81 | -------------------------------------------------------------------------------- /R/RevoBenchmark.R: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015, Andrie de Vries 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining 4 | # a copy of this software and associated documentation files (the 5 | # "Software"), to deal in the Software without restriction, including 6 | # without limitation the rights to use, copy, modify, merge, publish, 7 | # distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so, subject to 9 | # the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be 12 | # included in all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | 23 | # Set MKL threads if Revolution R Open or Revolution R Enterprise is available 24 | 25 | 26 | #' Returns the system time for a modified version of the Urbanek benchmarks. 27 | #' 28 | #' @param threads Number of Intel MKL threads to use if availalbe. Tests for the presence of the packages \code{RevoUtilsMath} or \code{RevoBase} and sets the number of threads using \code{RevoUtilsMath::setMKLthreads} 29 | #' @param show.message If TRUE, shows interim results as console messages 30 | #' @param scale.factor A numeric value that scales the size of the data up or down. The default value of 1 has data sizes that yields a runtime of ~2 seconds per test on an 8-core machine with the MKL available. For quick and easy testing, reduce the \code{scale.factor} to less than 1. (The primary use case for low \code{scale.factor} is to reduce the unit testing time when testing the package itself on CRAN.) To scale out the test data, use \code{scale.factor} of greater than 1. 31 | #' 32 | #' @importFrom MASS lda 33 | #' @export 34 | #' @seealso plot.RevoBenchmark 35 | RevoBenchmark <- function(threads = 4, show.message = TRUE, scale.factor = 1){ 36 | 37 | getMKLthreads <- setMKLthreads <- NULL # Trick to pass R CMD check 38 | 39 | rescale <- function(x){ 40 | round(x * sqrt(scale.factor)) 41 | } 42 | 43 | 44 | 45 | runUrbanek <- function(threads){ 46 | # if(requireNamespace("RevoUtilsMath", quietly = TRUE) || requireNamespace("RevoBase", quietly = TRUE)){ 47 | # if("RevoUtilsMath" %in% available.packages()[, "Package"]){ 48 | if("package:RevoUtilsMath" %in% search()){ 49 | oldThreads <- getMKLthreads() 50 | setMKLthreads(threads) 51 | on.exit(setMKLthreads(oldThreads)) 52 | } 53 | 54 | test.names <- c("Matrix multiplication", 55 | "Cholesky factorization", 56 | "QR decomposition", 57 | "Singular value decomposition", 58 | "Principal component analysis", 59 | "Linear discriminant analysis") 60 | elapsed.time <- rep(NA, length(test.names)) 61 | names(elapsed.time) <- test.names 62 | 63 | mssg <- function(i, message = show.message){ 64 | if(message){ 65 | msg <- paste( 66 | format(test.names[i], width = max(nchar(test.names)) + 3), 67 | format(elapsed.time[i], signif = 5, width = 7), 68 | sep = ": " 69 | ) 70 | message(msg) 71 | } 72 | } 73 | 74 | if(show.message) message("\nThreads:", threads) 75 | 76 | 77 | # Initialization 78 | 79 | set.seed (1) 80 | m <- rescale(10000) 81 | n <- rescale(5000) 82 | A <- matrix (runif (m*n),m,n) 83 | 84 | # Matrix multiply 85 | elapsed.time[1] <- system.time (B <- crossprod(A))[3] 86 | mssg(1) 87 | 88 | # Cholesky Factorization 89 | elapsed.time[2] <- system.time (C <- chol(B))[3] 90 | mssg(2) 91 | 92 | # QR decomposition 93 | m <- rescale(5000) 94 | n <- rescale(1000) 95 | Q <- matrix (runif (m*n),m,n) 96 | elapsed.time[3] <- system.time( qr(Q) )[3] 97 | mssg(3) 98 | 99 | 100 | # Singular Value Decomposition 101 | m <- rescale(10000) 102 | n <- rescale(2000) 103 | A <- matrix (runif (m*n), m, n) 104 | elapsed.time[4] <- system.time (S <- svd (A,nu=0,nv=0))[3] 105 | mssg(4) 106 | 107 | # Principal Components Analysis 108 | m <- rescale(2000) 109 | n <- rescale(2000) 110 | A <- matrix (runif (m*n), m, n) 111 | elapsed.time[5] <- system.time (P <- prcomp(A))[3] 112 | mssg(5) 113 | 114 | 115 | # Linear Discriminant Analysis 116 | g <- 5 117 | m <- rescale(3000) 118 | n <- rescale(1000) 119 | k <- round (m/2) 120 | A <- matrix (runif (m*n), m, n) 121 | A <- data.frame (A, fac = sample (LETTERS[1:g], m, replace = TRUE)) 122 | train <- sample(1:m, k) 123 | elapsed.time[6] <- system.time (L <- MASS::lda(fac ~., data = A, prior = rep(1, g)/g, subset = train))[3] 124 | mssg(6) 125 | 126 | elapsed.time 127 | } 128 | if(!"package:RevoUtilsMath" %in% search()) threads <- threads[1] 129 | ret <- lapply(threads, runUrbanek) 130 | ret <- do.call(cbind, ret) 131 | 132 | rVersion <- if(exists("Revo.version")) { 133 | revo <- with(Revo.version, paste(major, minor, sep = ".")) 134 | if(compareVersion(revo, "3.2.3") >= 0) "Microsoft R Open" else "RRO" 135 | } else { 136 | "R" 137 | } 138 | 139 | rVersionList <- if(exists("Revo.version")) Revo.version else R.version 140 | rVersion <- sprintf("%s-%s.%s", rVersion, rVersionList$major, rVersionList$minor) 141 | colnames(ret) <- sprintf("%s\ (%s thread%s)", rVersion, threads, ifelse(threads > 1, "s", "")) 142 | class(ret) <- c("RevoBenchmark", "matrix") 143 | Revo.version <- NULL # trick to pass R CMD check 144 | attr(ret, "R.version") <- list(R.version = R.version, 145 | Revo.version = if(exists("Revo.version")) Revo.version else NA 146 | ) 147 | ret 148 | 149 | } 150 | 151 | 152 | print.RevoBenchmark <- function(x, digits = 2, ...){ 153 | attr(x, "R.version") <- NULL 154 | x <- unclass(x) 155 | NextMethod(x, digits = digits) 156 | } 157 | 158 | 159 | 160 | --------------------------------------------------------------------------------