├── .gitignore ├── data ├── datalist ├── sb.RData ├── tdat.rda ├── quantSB.rda └── soneson2016.rda ├── docs ├── favicon.ico ├── reference │ ├── figures │ │ └── sbhex.png │ └── BDData-set.html ├── articles │ ├── summarizedbenchmark-figure1.png │ ├── summarizedbenchmark-figure2.png │ ├── QuantificationBenchmark_files │ │ └── figure-html │ │ │ └── pcaPlot-1.png │ ├── CaseStudy-RNAseqQuantification_files │ │ └── figure-html │ │ │ └── pcaPlot-1.png │ ├── SummarizedBenchmark-FullCaseStudy_files │ │ └── figure-html │ │ │ ├── unnamed-chunk-13-1.png │ │ │ └── unnamed-chunk-14-1.png │ └── CaseStudy-SingleCellSimulation_files │ │ └── figure-html │ │ ├── plot-performace-metrics-1.png │ │ ├── plot-sim-result-comparison-1.png │ │ └── plot-sim-result-comparison-2.png ├── pkgdown.yml ├── link.svg ├── docsearch.js ├── pkgdown.js └── pkgdown.css ├── man ├── figures │ └── sbhex.png ├── BDData-show.Rd ├── BDMethod-show.Rd ├── BenchDesign-show.Rd ├── BDMethodList-show.Rd ├── sb.Rd ├── modmethod.Rd ├── BDData-class.Rd ├── BDMethodList-class.Rd ├── allSB.Rd ├── groundTruths.Rd ├── compareBDMethod.Rd ├── tdat.Rd ├── plotROC.Rd ├── groundTruths-setter.Rd ├── availableMetrics.Rd ├── mcols-setter.Rd ├── tidyUpMetrics.Rd ├── BDData-setter.Rd ├── hashBDData.Rd ├── assayNames-setter.Rd ├── BDMethodList-setter.Rd ├── compareBDData.Rd ├── BDMethod-class.Rd ├── plotMethodsOverlap.Rd ├── printMethod.Rd ├── dropMethod.Rd ├── performanceMetrics-setter.Rd ├── performanceMetrics.Rd ├── BenchDesign-class.Rd ├── BDMethod-setter.Rd ├── addPerformanceMetric.Rd ├── SummarizedBenchmark-class.Rd ├── tidyBDMethod.Rd ├── BDData.Rd ├── BDMethodList.Rd ├── modifyMethod.Rd ├── BenchDesign.Rd ├── compareBenchDesigns.Rd ├── SummarizedBenchmark.Rd ├── expandMethod.Rd ├── BDMethod.Rd ├── updateBench.Rd ├── estimateMetrics.Rd ├── addMethod.Rd └── buildBench.Rd ├── tests ├── testthat.R └── testthat │ ├── test_plottingFunctions.R │ ├── test_SummarizedBenchmark.R │ ├── test_addPerformanceFunction.R │ ├── test_metricsRerun.R │ ├── test_estimatePerformanceMetrics.R │ ├── test-updateBench.R │ └── test-BDMethod.R ├── vignettes ├── summarizedbenchmark-figure1.png ├── summarizedbenchmark-figure2.png ├── library.bib └── Feature-ErrorHandling.Rmd ├── .Rbuildignore ├── SummarizedBenchmark.Rproj ├── R ├── data-soneson2016.R ├── data-quant.R ├── data-tdat.R ├── BDMethodList-accessors.R ├── BDData-show.R ├── BDMethodList-show.R ├── printMethod.R ├── dropMethod.R ├── BenchDesign-accessors.R ├── BenchDesign-show.R ├── BDMethod-show.R ├── BenchDesign-constructor.R ├── DefaultMetrics.R ├── PlottingFunctions.R ├── BDMethod-constructor.R ├── BDData-constructor.R ├── BDMethodList-constructor.R ├── modifyMethod.R ├── SummarizedBenckmark-constructor.R ├── BDMethod-tidy.R ├── SummarizedBenckmark-accessors.R ├── addMethod.R ├── printUpdateBench.R └── expandMethod.R ├── inst ├── CITATION └── NEWS.Rd ├── DESCRIPTION ├── README.md ├── README.Rmd ├── NAMESPACE └── _pkgdown.yml /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | README.html -------------------------------------------------------------------------------- /data/datalist: -------------------------------------------------------------------------------- 1 | quantSB: allSB 2 | sb 3 | soneson2016: truthdat txi 4 | tdat 5 | -------------------------------------------------------------------------------- /data/sb.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/data/sb.RData -------------------------------------------------------------------------------- /data/tdat.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/data/tdat.rda -------------------------------------------------------------------------------- /data/quantSB.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/data/quantSB.rda -------------------------------------------------------------------------------- /docs/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/favicon.ico -------------------------------------------------------------------------------- /data/soneson2016.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/data/soneson2016.rda -------------------------------------------------------------------------------- /man/figures/sbhex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/man/figures/sbhex.png -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- 1 | library("testthat") 2 | library("SummarizedBenchmark") 3 | 4 | test_check("SummarizedBenchmark") 5 | -------------------------------------------------------------------------------- /docs/reference/figures/sbhex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/reference/figures/sbhex.png -------------------------------------------------------------------------------- /vignettes/summarizedbenchmark-figure1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/vignettes/summarizedbenchmark-figure1.png -------------------------------------------------------------------------------- /vignettes/summarizedbenchmark-figure2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/vignettes/summarizedbenchmark-figure2.png -------------------------------------------------------------------------------- /docs/articles/summarizedbenchmark-figure1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/articles/summarizedbenchmark-figure1.png -------------------------------------------------------------------------------- /docs/articles/summarizedbenchmark-figure2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/articles/summarizedbenchmark-figure2.png -------------------------------------------------------------------------------- /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^docs$ 2 | ^.travis\.yml$ 3 | ^_pkgdown\.yml$ 4 | ^.*\.Rproj$ 5 | ^\.Rproj\.user$ 6 | .Rhistory 7 | .DS_Store 8 | vignettes/site-* 9 | vignettes/*_cache 10 | 11 | -------------------------------------------------------------------------------- /docs/articles/QuantificationBenchmark_files/figure-html/pcaPlot-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/articles/QuantificationBenchmark_files/figure-html/pcaPlot-1.png -------------------------------------------------------------------------------- /docs/articles/CaseStudy-RNAseqQuantification_files/figure-html/pcaPlot-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/articles/CaseStudy-RNAseqQuantification_files/figure-html/pcaPlot-1.png -------------------------------------------------------------------------------- /docs/articles/SummarizedBenchmark-FullCaseStudy_files/figure-html/unnamed-chunk-13-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/articles/SummarizedBenchmark-FullCaseStudy_files/figure-html/unnamed-chunk-13-1.png -------------------------------------------------------------------------------- /docs/articles/SummarizedBenchmark-FullCaseStudy_files/figure-html/unnamed-chunk-14-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/articles/SummarizedBenchmark-FullCaseStudy_files/figure-html/unnamed-chunk-14-1.png -------------------------------------------------------------------------------- /docs/articles/CaseStudy-SingleCellSimulation_files/figure-html/plot-performace-metrics-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/articles/CaseStudy-SingleCellSimulation_files/figure-html/plot-performace-metrics-1.png -------------------------------------------------------------------------------- /docs/articles/CaseStudy-SingleCellSimulation_files/figure-html/plot-sim-result-comparison-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/articles/CaseStudy-SingleCellSimulation_files/figure-html/plot-sim-result-comparison-1.png -------------------------------------------------------------------------------- /docs/articles/CaseStudy-SingleCellSimulation_files/figure-html/plot-sim-result-comparison-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/areyesq89/SummarizedBenchmark/HEAD/docs/articles/CaseStudy-SingleCellSimulation_files/figure-html/plot-sim-result-comparison-2.png -------------------------------------------------------------------------------- /man/BDData-show.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/BDData-show.R 3 | \docType{methods} 4 | \name{show,BDData-method} 5 | \alias{show,BDData-method} 6 | \title{Show BDData object} 7 | \usage{ 8 | \S4method{show}{BDData}(object) 9 | } 10 | \arguments{ 11 | \item{object}{BDData object to show} 12 | } 13 | \value{ 14 | Print description of BDData object to console 15 | } 16 | \description{ 17 | Show BDData object 18 | } 19 | -------------------------------------------------------------------------------- /man/BDMethod-show.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/BDMethod-show.R 3 | \docType{methods} 4 | \name{show,BDMethod-method} 5 | \alias{show,BDMethod-method} 6 | \title{Show BDMethod object} 7 | \usage{ 8 | \S4method{show}{BDMethod}(object) 9 | } 10 | \arguments{ 11 | \item{object}{BDMethod object to show} 12 | } 13 | \value{ 14 | Print description of BDMethod object to console 15 | } 16 | \description{ 17 | Show BDMethod object 18 | } 19 | -------------------------------------------------------------------------------- /SummarizedBenchmark.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | 18 | BuildType: Package 19 | PackageUseDevtools: Yes 20 | PackageInstallArgs: --no-multiarch --with-keep.source 21 | PackageRoxygenize: rd,collate,namespace 22 | -------------------------------------------------------------------------------- /man/BenchDesign-show.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/BenchDesign-show.R 3 | \docType{methods} 4 | \name{show,BenchDesign-method} 5 | \alias{show,BenchDesign-method} 6 | \title{Show BenchDesign object} 7 | \usage{ 8 | \S4method{show}{BenchDesign}(object) 9 | } 10 | \arguments{ 11 | \item{object}{BenchDesign object to show} 12 | } 13 | \value{ 14 | Print description of BenchDesign object to console 15 | } 16 | \description{ 17 | Show BenchDesign object 18 | } 19 | -------------------------------------------------------------------------------- /man/BDMethodList-show.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/BDMethodList-show.R 3 | \docType{methods} 4 | \name{show,BDMethodList-method} 5 | \alias{show,BDMethodList-method} 6 | \title{Show BDMethodList object} 7 | \usage{ 8 | \S4method{show}{BDMethodList}(object) 9 | } 10 | \arguments{ 11 | \item{object}{BDMethodList object to show} 12 | } 13 | \value{ 14 | Print description of BDMethodList object to console 15 | } 16 | \description{ 17 | Show BDMethodList object 18 | } 19 | -------------------------------------------------------------------------------- /R/data-soneson2016.R: -------------------------------------------------------------------------------- 1 | #' SummarizedBenchmark example 2 | #' 3 | #' @name sb 4 | #' @description This object contains the example data from the \code{iCOBRA} package reformatted as a 5 | #' \code{SummarizedBenchmark} object. It consists of differential expression results from \code{DESeq2} 6 | #' \code{edgeR} and \code{limma}-voom. 7 | #' @source Example data from the \code{iCOBRA} package. 8 | #' @aliases truthdat txi 9 | #' @docType data 10 | #' @keywords data SummarizedBenchmark 11 | #' @examples 12 | #' data(sb) 13 | NULL 14 | -------------------------------------------------------------------------------- /docs/pkgdown.yml: -------------------------------------------------------------------------------- 1 | pandoc: 2.1.1 2 | pkgdown: 1.3.0 3 | pkgdown_sha: ~ 4 | articles: 5 | CaseStudy-RNAseqQuantification: CaseStudy-RNAseqQuantification.html 6 | CaseStudy-SingleCellSimulation: CaseStudy-SingleCellSimulation.html 7 | Feature-ErrorHandling: Feature-ErrorHandling.html 8 | Feature-Iterative: Feature-Iterative.html 9 | Feature-Parallel: Feature-Parallel.html 10 | SummarizedBenchmark-ClassDetails: SummarizedBenchmark-ClassDetails.html 11 | SummarizedBenchmark-FullCaseStudy: SummarizedBenchmark-FullCaseStudy.html 12 | SummarizedBenchmark-Introduction: SummarizedBenchmark-Introduction.html 13 | 14 | -------------------------------------------------------------------------------- /R/data-quant.R: -------------------------------------------------------------------------------- 1 | #' SummarizedBenchmark object of isoform quantification results 2 | #' 3 | #' @name allSB 4 | #' @aliases allSB,quantSB 5 | #' @description This object is a \code{SummarizedBenchmark} object containing isoform quantifications from salmon, sailfish and kallisto 6 | #' from 4 mouse samples (2 hearts and 2 brains) part of the Mouse BodyMap. Its generation is described in one of the vignettes of this package. 7 | #' @source Mouse BodyMap (Li et al, 2014). SRA accession numbers SRR5273705, SRR5273689, SRR5273699 and SRR5273683. 8 | #' @docType data 9 | #' @keywords data 10 | #' @examples 11 | #' data(quantSB) 12 | NULL 13 | -------------------------------------------------------------------------------- /man/sb.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/data-soneson2016.R 3 | \docType{data} 4 | \name{sb} 5 | \alias{sb} 6 | \alias{truthdat} 7 | \alias{txi} 8 | \title{SummarizedBenchmark example} 9 | \source{ 10 | Example data from the \code{iCOBRA} package. 11 | } 12 | \description{ 13 | This object contains the example data from the \code{iCOBRA} package reformatted as a 14 | \code{SummarizedBenchmark} object. It consists of differential expression results from \code{DESeq2} 15 | \code{edgeR} and \code{limma}-voom. 16 | } 17 | \examples{ 18 | data(sb) 19 | } 20 | \keyword{SummarizedBenchmark} 21 | \keyword{data} 22 | -------------------------------------------------------------------------------- /man/modmethod.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/modifyMethod.R 3 | \name{.modmethod} 4 | \alias{.modmethod} 5 | \title{Modify BenchDesign Method} 6 | \usage{ 7 | .modmethod(m, q, .overwrite) 8 | } 9 | \arguments{ 10 | \item{m}{method} 11 | 12 | \item{q}{quosure list of new parameters} 13 | 14 | \item{.overwrite}{logical whether to overwrite parameters} 15 | } 16 | \value{ 17 | modified method. 18 | } 19 | \description{ 20 | Given a method defined in a BenchDesign, this helper function 21 | returns a modified method with new parameters defined as a 22 | list of quosures. 23 | } 24 | \author{ 25 | Patrick Kimes 26 | } 27 | \keyword{internal} 28 | -------------------------------------------------------------------------------- /man/BDData-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllClasses.R 3 | \docType{class} 4 | \name{BDData-class} 5 | \alias{BDData-class} 6 | \title{BDData class} 7 | \description{ 8 | Container for data in a BenchDesign object. 9 | } 10 | \section{Slots}{ 11 | 12 | \describe{ 13 | \item{\code{data}}{a list or MD5 hash of the data.} 14 | 15 | \item{\code{type}}{a character string indicating whether the data slot 16 | contains the 'data' or a 'md5hash' of the data.} 17 | }} 18 | 19 | \seealso{ 20 | \code{\link{BDData}}, \code{\link{BenchDesign-class}}, \code{\link{BDMethod-class}}, \code{\link{BDMethodList-class}} 21 | } 22 | \author{ 23 | Patrick Kimes 24 | } 25 | -------------------------------------------------------------------------------- /man/BDMethodList-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllClasses.R 3 | \docType{class} 4 | \name{BDMethodList-class} 5 | \alias{BDMethodList-class} 6 | \title{BDMethodList class} 7 | \description{ 8 | Extension of the SimpleList class to contain a list of BDMethod 9 | objects. The class serves as the primary container for the set of 10 | methods in the BenchDesign class. 11 | 12 | New BDMethodList objects can be created using the 13 | \code{\link{BDMethodList}} constructor. 14 | } 15 | \seealso{ 16 | \code{\link{BDMethodList}}, \code{\link{BenchDesign-class}}, \code{\link{BDMethod-class}}, \code{\link{BDData-class}} 17 | } 18 | \author{ 19 | Patrick Kimes 20 | } 21 | -------------------------------------------------------------------------------- /inst/CITATION: -------------------------------------------------------------------------------- 1 | citEntry(entry="article", 2 | title = "Reproducible and replicable comparisons using SummarizedBenchmark.", 3 | author = personList( 4 | as.person("Patrick Kimes"), 5 | as.person("Alejandro Reyes")), 6 | year = 2019, 7 | journal = "Bioinformatics", 8 | volume = 35, 9 | issue = 1, 10 | pages = "137", 11 | doi = "10.1093/bioinformatics/bty627", 12 | textVersion = 13 | paste("Patrick Kimes and Alejandro Reyes (2019)", 14 | "Reproducible and replicable comparisons using SummarizedBenchmark.", 15 | "Bioinformatics (2019) 35(1):137. doi:10.1093/bioinformatics/bty627" ) ) 16 | -------------------------------------------------------------------------------- /R/data-tdat.R: -------------------------------------------------------------------------------- 1 | #' Example data.frame containing results for 50 two-sample t-tests. 2 | #' 3 | #' @format 4 | #' a data.frame that contains the results of 50 simulated two-sample 5 | #' t-tests, with each row corresponding to an independent test. The 6 | #' data.frame includes the following 5 columns: 7 | #' 1. H = binary 0/1 whether data for the test was simulated under the null (0) or alternative (1) 8 | #' 2. test_statistic = test-statistics of the t-test 9 | #' 3. effect_size = mean difference between the two sample groups 10 | #' 4. pval = p-value of the t-test 11 | #' 5. SE = standard error of the t-test 12 | #' 13 | #' @name tdat 14 | #' @docType data 15 | #' @keywords data 16 | #' @examples 17 | #' data(tdat) 18 | NULL 19 | -------------------------------------------------------------------------------- /man/allSB.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/data-quant.R 3 | \docType{data} 4 | \name{allSB} 5 | \alias{allSB} 6 | \alias{allSB,quantSB} 7 | \title{SummarizedBenchmark object of isoform quantification results} 8 | \source{ 9 | Mouse BodyMap (Li et al, 2014). SRA accession numbers SRR5273705, SRR5273689, SRR5273699 and SRR5273683. 10 | } 11 | \description{ 12 | This object is a \code{SummarizedBenchmark} object containing isoform quantifications from salmon, sailfish and kallisto 13 | from 4 mouse samples (2 hearts and 2 brains) part of the Mouse BodyMap. Its generation is described in one of the vignettes of this package. 14 | } 15 | \examples{ 16 | data(quantSB) 17 | } 18 | \keyword{data} 19 | -------------------------------------------------------------------------------- /man/groundTruths.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, 3 | % R/SummarizedBenckmark-accessors.R 4 | \docType{methods} 5 | \name{groundTruths} 6 | \alias{groundTruths} 7 | \alias{groundTruths,SummarizedBenchmark-method} 8 | \title{Get ground truths in SummarizedBenchmark object} 9 | \usage{ 10 | groundTruths(object, ...) 11 | 12 | \S4method{groundTruths}{SummarizedBenchmark}(object, ...) 13 | } 14 | \arguments{ 15 | \item{object}{a \code{SummarizedBenchmark} object.} 16 | 17 | \item{...}{further arguments, perhaps used by methods.} 18 | } 19 | \value{ 20 | modified BenchDesign object 21 | } 22 | \description{ 23 | Method to get groundTruths in SummarizedBenchmark object. 24 | } 25 | \author{ 26 | Alejandro Reyes 27 | } 28 | -------------------------------------------------------------------------------- /docs/link.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 8 | 12 | 13 | -------------------------------------------------------------------------------- /man/compareBDMethod.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/compareBenchDesigns.R 3 | \name{compareBDMethod} 4 | \alias{compareBDMethod} 5 | \title{Compare BDMethod objects} 6 | \usage{ 7 | compareBDMethod(x, y) 8 | } 9 | \arguments{ 10 | \item{x}{a \code{BDMethod} object} 11 | 12 | \item{y}{a \code{BDMethod} object} 13 | } 14 | \value{ 15 | logical value indicating whether the two objects produced the same 16 | meta data. 17 | } 18 | \description{ 19 | Simple comparison of two BDMethod objects based on 20 | meta data. 21 | } 22 | \examples{ 23 | bdm1 <- BDMethod(stats::rnorm, params = rlang::quos(n = 100)) 24 | bdm2 <- BDMethod(stats::rt, params = rlang::quos(n = 100, df = 1)) 25 | 26 | compareBDMethod(bdm1, bdm2) 27 | 28 | } 29 | \seealso{ 30 | \code{\link{compareBenchDesigns}} 31 | } 32 | \author{ 33 | Patrick Kimes 34 | } 35 | -------------------------------------------------------------------------------- /man/tdat.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/data-tdat.R 3 | \docType{data} 4 | \name{tdat} 5 | \alias{tdat} 6 | \title{Example data.frame containing results for 50 two-sample t-tests.} 7 | \format{a data.frame that contains the results of 50 simulated two-sample 8 | t-tests, with each row corresponding to an independent test. The 9 | data.frame includes the following 5 columns: 10 | 1. H = binary 0/1 whether data for the test was simulated under the null (0) or alternative (1) 11 | 2. test_statistic = test-statistics of the t-test 12 | 3. effect_size = mean difference between the two sample groups 13 | 4. pval = p-value of the t-test 14 | 5. SE = standard error of the t-test} 15 | \description{ 16 | Example data.frame containing results for 50 two-sample t-tests. 17 | } 18 | \examples{ 19 | data(tdat) 20 | } 21 | \keyword{data} 22 | -------------------------------------------------------------------------------- /man/plotROC.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/PlottingFunctions.R 3 | \name{plotROC} 4 | \alias{plotROC} 5 | \title{Plot ROC curve for SummarizedBenchmark object} 6 | \usage{ 7 | plotROC(object, assay = "qvalue") 8 | } 9 | \arguments{ 10 | \item{object}{A \code{\link{SummarizedBenchmark}} object.} 11 | 12 | \item{assay}{An assay name.} 13 | } 14 | \value{ 15 | A ggplot object. 16 | } 17 | \description{ 18 | This function inputs a \code{\link{SummarizedBenchmark}} object, looks 19 | for an assay called 'qvalue' and plots receiver operating characteristic curves 20 | for each of the methods to benchmark. 21 | } 22 | \examples{ 23 | data(sb) 24 | \dontrun{ 25 | plotROC(sb) 26 | } 27 | 28 | } 29 | \seealso{ 30 | \code{\link{plotMethodsOverlap}}, \code{\link{estimatePerformanceMetrics}} 31 | } 32 | \author{ 33 | Alejandro Reyes 34 | } 35 | -------------------------------------------------------------------------------- /man/groundTruths-setter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, 3 | % R/SummarizedBenckmark-accessors.R 4 | \docType{methods} 5 | \name{groundTruths<-} 6 | \alias{groundTruths<-} 7 | \alias{groundTruths<-,SummarizedBenchmark-method} 8 | \title{Set ground truths in SummarizedBenchmark object} 9 | \usage{ 10 | groundTruths(object, ...) <- value 11 | 12 | \S4method{groundTruths}{SummarizedBenchmark}(object, ...) <- value 13 | } 14 | \arguments{ 15 | \item{object}{a \code{SummarizedBenchmark} object.} 16 | 17 | \item{...}{futher arguments, perhaps used by methods.} 18 | 19 | \item{value}{replacement set of ground truths.} 20 | } 21 | \value{ 22 | modified BenchDesign object 23 | } 24 | \description{ 25 | Method to set groundTruths in SummarizedBenchmark object. 26 | } 27 | \author{ 28 | Alejandro Reyes, Patrick Kimes 29 | } 30 | -------------------------------------------------------------------------------- /man/availableMetrics.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/DefaultMetrics.R 3 | \name{availableMetrics} 4 | \alias{availableMetrics} 5 | \title{List pre-defined metrics for SummarizedBenchmark objects} 6 | \usage{ 7 | availableMetrics() 8 | } 9 | \value{ 10 | A data.frame summarizing the default performance metrics provided in this package. 11 | } 12 | \description{ 13 | This function returns a data frame summarizing the default performance metrics provided in this package. 14 | The data.frame contains three columns, \code{functions} is the name of the performance metric, \code{description} 15 | is longer description of the performance metric and \code{requiredTruth} is logical depending on whether the 16 | performance metrics require ground truths. 17 | } 18 | \examples{ 19 | availableMetrics() 20 | 21 | } 22 | \author{ 23 | Alejandro Reyes 24 | } 25 | -------------------------------------------------------------------------------- /R/BDMethodList-accessors.R: -------------------------------------------------------------------------------- 1 | #' @rdname BDMethod-setter 2 | #' @exportMethod "BDMethod<-" 3 | setReplaceMethod("BDMethod", 4 | signature(x = "BDMethodList", i = "character", value = "BDMethod"), 5 | function (x, i, value) { 6 | x[[i]] <- value 7 | x 8 | }) 9 | 10 | #' @rdname BDMethod-setter 11 | #' @exportMethod "BDMethod<-" 12 | setReplaceMethod("BDMethod", 13 | signature(x = "BDMethodList", i = "character", value = "NULL"), 14 | function (x, i, value) { 15 | x[[i]] <- value 16 | x 17 | }) 18 | 19 | #' @rdname BDMethod 20 | #' @exportMethod "BDMethod" 21 | setMethod("BDMethod", 22 | signature(x = "BDMethodList"), 23 | function(x, i = 1) { 24 | stopifnot(is(i, "character") || is(i, "numeric")) 25 | x[[i]] 26 | }) 27 | -------------------------------------------------------------------------------- /man/mcols-setter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/SummarizedBenckmark-accessors.R 3 | \docType{methods} 4 | \name{mcols<-,SummarizedBenchmark-method} 5 | \alias{mcols<-,SummarizedBenchmark-method} 6 | \title{Set meta data columns in SummarizedBenchmark object} 7 | \usage{ 8 | \S4method{mcols}{SummarizedBenchmark}(x, ...) <- value 9 | } 10 | \arguments{ 11 | \item{x}{A \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object.} 12 | 13 | \item{...}{Futher arguments, perhaps used by methods.} 14 | 15 | \item{value}{A DataFrame of meta data.} 16 | } 17 | \value{ 18 | Modfied \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. 19 | } 20 | \description{ 21 | Modifies the \code{mcols} slot of a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 22 | object. 23 | } 24 | \seealso{ 25 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 26 | } 27 | \author{ 28 | Alejandro Reyes 29 | } 30 | -------------------------------------------------------------------------------- /tests/testthat/test_plottingFunctions.R: -------------------------------------------------------------------------------- 1 | context( "plottingFunctions" ) 2 | test_that( "test that 'plotROC' and 'plotOverlaps' behave as expected", { 3 | data( sb ) 4 | sb <- addPerformanceMetric( sb, evalMetric=c("rejections", "TPR", "TNR", "FDR", "FNR"), assay="qvalue" ) 5 | expect_error( plotROC( 1:10 ) ) 6 | sb2 <- sb 7 | groundTruths( sb2 ) <- NULL 8 | expect_error( plotROC( sb2 ) ) 9 | sb2 <- sb 10 | assayNames( sb2 ) <- paste0( "assay", seq_along( assayNames( sb2 ) ) ) 11 | expect_error( plotROC( sb2 ) ) 12 | expect_error( plotMethodsOverlap( sb2 ) ) 13 | expect_error( plotMethodsOverlap( object=1:10 ) ) 14 | expect_error( plotMethodsOverlap( sb, alpha="a" ) ) 15 | expect_error( plotMethodsOverlap( sb, alpha=3 ) ) 16 | expect_identical( 17 | plotROC( sb )$geom$objname, 18 | plotROC( sb2, assay="assay1" )$geom$objname ) 19 | expect_is( plotMethodsOverlap( sb ), "upset" ) 20 | expect_is( plotMethodsOverlap( sb2, assay="assay1" ), "upset" ) 21 | } ) 22 | -------------------------------------------------------------------------------- /man/tidyUpMetrics.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/PerformanceMetrics.R 3 | \name{tidyUpMetrics} 4 | \alias{tidyUpMetrics} 5 | \title{Tidy up performance metrics in SummarizedBenchmark object} 6 | \usage{ 7 | tidyUpMetrics(object) 8 | } 9 | \arguments{ 10 | \item{object}{A \code{\link{SummarizedBenchmark}} object.} 11 | } 12 | \value{ 13 | A tidy \code{data.frame} 14 | } 15 | \description{ 16 | This function takes as input a \code{SummarizedBenchmark} object, extracts the 17 | estimated performance metrics and reformats them into a long-formated data frame. 18 | } 19 | \examples{ 20 | data( "sb", package="SummarizedBenchmark" ) 21 | sb <- estimateMetricsForAssay( sb, assay="qvalue", evalMetric="rejections", 22 | evalFunction=function( query, truth, alpha=0.1 ){ 23 | sum( query < alpha ) 24 | }, 25 | addColData=TRUE ) 26 | tidyUpMetrics( sb ) 27 | 28 | } 29 | \seealso{ 30 | \code{\link{estimatePerformanceMetrics}} 31 | } 32 | \author{ 33 | Alejandro Reyes 34 | } 35 | -------------------------------------------------------------------------------- /man/BDData-setter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/BenchDesign-accessors.R 3 | \docType{methods} 4 | \name{BDData<-} 5 | \alias{BDData<-} 6 | \alias{BDData<-,BenchDesign,BDDataOrNULL-method} 7 | \title{Set data in BenchDesign object} 8 | \usage{ 9 | BDData(x) <- value 10 | 11 | \S4method{BDData}{BenchDesign,BDDataOrNULL}(x) <- value 12 | } 13 | \arguments{ 14 | \item{x}{\code{\link[=BenchDesign-class]{BenchDesign}} object.} 15 | 16 | \item{value}{\code{\link[=BDData-class]{BDData}} or \code{NULL}.} 17 | } 18 | \value{ 19 | modified BenchDesign object 20 | } 21 | \description{ 22 | Adds, removes or replaces \code{\link[=BDData-class]{BDData}} in 23 | \code{\link[=BenchDesign-class]{BenchDesign}} object. 24 | Data can be removed by setting the value to \code{NULL}. 25 | } 26 | \examples{ 27 | bd <- BenchDesign() 28 | BDData(bd) <- BDData(data.frame(x1 = runif(5))) 29 | bd 30 | 31 | } 32 | \seealso{ 33 | \code{\link{BDData}} 34 | } 35 | \author{ 36 | Patrick Kimes 37 | } 38 | -------------------------------------------------------------------------------- /man/hashBDData.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/BDData-constructor.R 3 | \docType{methods} 4 | \name{hashBDData} 5 | \alias{hashBDData} 6 | \alias{hashBDData,BDData-method} 7 | \alias{hashBDData,BenchDesign-method} 8 | \title{Hash data in BDData object} 9 | \usage{ 10 | hashBDData(object) 11 | 12 | \S4method{hashBDData}{BDData}(object) 13 | 14 | \S4method{hashBDData}{BenchDesign}(object) 15 | } 16 | \arguments{ 17 | \item{object}{a \code{BDData} or \code{BenchDesign} object} 18 | } 19 | \value{ 20 | an object of the same class as \code{object} with data 21 | converted to a MD5 hash. 22 | } 23 | \description{ 24 | Repliaces data stored in a \code{\link[=BDData-class]{BDData}} object 25 | with the MD5 hash of the data. If the data was already a MD5 hash, the 26 | original object is returned unchanged. The method can be called directly 27 | on \code{\link[=BenchDesign-class]{BenchDesign}} objects to hash the 28 | underlying data as well. 29 | } 30 | \author{ 31 | Patrick Kimes 32 | } 33 | -------------------------------------------------------------------------------- /man/assayNames-setter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/SummarizedBenckmark-accessors.R 3 | \docType{methods} 4 | \name{assayNames<-,SummarizedBenchmark,character-method} 5 | \alias{assayNames<-,SummarizedBenchmark,character-method} 6 | \title{Set assay names in SummarizedBenchmark object} 7 | \usage{ 8 | \S4method{assayNames}{SummarizedBenchmark,character}(x, ...) <- value 9 | } 10 | \arguments{ 11 | \item{x}{A \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object.} 12 | 13 | \item{...}{Futher arguments, perhaps used by methods.} 14 | 15 | \item{value}{A character vector.} 16 | } 17 | \value{ 18 | Modfied \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. 19 | } 20 | \description{ 21 | Modifies the assay names of a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 22 | object. 23 | } 24 | \examples{ 25 | data(sb) 26 | assayNames(sb)[2] <- "log2FC" 27 | 28 | } 29 | \seealso{ 30 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 31 | } 32 | \author{ 33 | Alejandro Reyes 34 | } 35 | -------------------------------------------------------------------------------- /man/BDMethodList-setter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/BenchDesign-accessors.R 3 | \docType{methods} 4 | \name{BDMethodList<-} 5 | \alias{BDMethodList<-} 6 | \alias{BDMethodList<-,BenchDesign,BDMethodList-method} 7 | \title{Set method list in BenchDesign object} 8 | \usage{ 9 | BDMethodList(x) <- value 10 | 11 | \S4method{BDMethodList}{BenchDesign,BDMethodList}(x) <- value 12 | } 13 | \arguments{ 14 | \item{x}{BenchDesign object.} 15 | 16 | \item{value}{\code{\link[=BDMethodList-class]{BDMethodList}} list of methods.} 17 | } 18 | \value{ 19 | modified BenchDesign object 20 | } 21 | \description{ 22 | Replaces the \code{\link[=BDMethodList-class]{BDMethodList}} list of methods 23 | in a \code{\link[=BenchDesign-class]{BenchDesign}} object with a specified 24 | \code{\link[=BDMethodList-class]{BDMethodList}} object. 25 | } 26 | \examples{ 27 | bd <- BenchDesign() 28 | BDMethodList(bd) <- BDMethodList(avg = BDMethod(x = base::mean)) 29 | bd 30 | 31 | } 32 | \seealso{ 33 | \code{\link{BDMethod}}, \code{\link{BDMethodList}} 34 | } 35 | \author{ 36 | Patrick Kimes 37 | } 38 | -------------------------------------------------------------------------------- /man/compareBDData.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/compareBenchDesigns.R 3 | \name{compareBDData} 4 | \alias{compareBDData} 5 | \title{Compare BDData objects} 6 | \usage{ 7 | compareBDData(x, y) 8 | } 9 | \arguments{ 10 | \item{x}{a \code{BDData} or \code{BenchDesign} object} 11 | 12 | \item{y}{a \code{BDData} or \code{BenchDesign} object} 13 | } 14 | \value{ 15 | list of two values giving agreement of "data" and "type". 16 | } 17 | \description{ 18 | Simple comparison of two BDData objects based on 19 | comparing both type and data hash. 20 | } 21 | \examples{ 22 | ## compare data with same MD5 hash value 23 | bdd1 <- BDData(data.frame(x = 1:10)) 24 | bdd1h <- hashBDData(bdd1) 25 | compareBDData(bdd1, bdd1h) 26 | 27 | ## compare different data, both same type 28 | bdd2 <- BDData(data.frame(x = 2:11)) 29 | bdd2h <- hashBDData(bdd2) 30 | compareBDData(bdd1, bdd2) 31 | compareBDData(bdd1h, bdd2h) 32 | 33 | ## compare completely different data 34 | compareBDData(bdd1, bdd2h) 35 | 36 | } 37 | \seealso{ 38 | \code{\link{compareBenchDesigns}} 39 | } 40 | \author{ 41 | Patrick Kimes 42 | } 43 | -------------------------------------------------------------------------------- /tests/testthat/test_SummarizedBenchmark.R: -------------------------------------------------------------------------------- 1 | context("SummarizedBenchmark") 2 | test_that("test='SummarizedBenchmark' returns error messages", { 3 | library(iCOBRA) 4 | data(cobradata_example) 5 | assays <- list( 6 | qvalue=cobradata_example@padj, 7 | logFC=cobradata_example@score ) 8 | assays[["qvalue"]]$DESeq2 <- p.adjust(cobradata_example@pval$DESeq2, method="BH") 9 | groundTruth <- DataFrame( cobradata_example@truth[,c("status", "logFC")] ) 10 | colnames(groundTruth) <- names( assays ) 11 | colData <- DataFrame( method=colnames(assays[[1]]) ) 12 | groundTruth <- groundTruth[rownames(assays[[1]]),] 13 | expect_error( SummarizedBenchmark( 14 | assays=assays, 15 | colData=colData, 16 | groundTruth=groundTruth[1:10,] ) ) 17 | assays2 <- assays 18 | assays2[[1]] <- assays2[[1]][1:10,] 19 | expect_error( SummarizedBenchmark( 20 | assays=assays2, 21 | colData=colData, 22 | groundTruth=groundTruth ) ) 23 | assays2 <- assays 24 | assays2[[1]] <- assays2[[1]][,1:2] 25 | expect_error( SummarizedBenchmark( 26 | assays=assays2, 27 | colData=colData, 28 | groundTruth=groundTruth ) ) 29 | } ) 30 | -------------------------------------------------------------------------------- /man/BDMethod-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllClasses.R 3 | \docType{class} 4 | \name{BDMethod-class} 5 | \alias{BDMethod-class} 6 | \title{BDMethod class} 7 | \description{ 8 | Container for individual methods to be compared as part of a benchmark 9 | experiment defined in a BenchDesign object. In the SummarizedBenchmark 10 | framework, methods are defined by a unique combination of functions, 11 | parameters, and any relevant meta data. 12 | 13 | New BDMethod objects can be created using the \code{\link{BDMethod}} 14 | constructor. 15 | } 16 | \section{Slots}{ 17 | 18 | \describe{ 19 | \item{\code{f}}{a function to be benchmarked} 20 | 21 | \item{\code{fc}}{a captured expression of the function \code{f}} 22 | 23 | \item{\code{params}}{a list of quosures specifying function parameters} 24 | 25 | \item{\code{post}}{a list of functions to be applied to the output of \code{f}} 26 | 27 | \item{\code{meta}}{a list of meta data} 28 | }} 29 | 30 | \seealso{ 31 | \code{\link{BDMethod}}, \code{\link{BenchDesign-class}}, \code{\link{BDMethodList-class}}, \code{\link{BDData-class}} 32 | } 33 | \author{ 34 | Patrick Kimes 35 | } 36 | -------------------------------------------------------------------------------- /man/plotMethodsOverlap.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/PlottingFunctions.R 3 | \name{plotMethodsOverlap} 4 | \alias{plotMethodsOverlap} 5 | \title{Plot UpSetR for SummarizedBenchmark object} 6 | \usage{ 7 | plotMethodsOverlap(object, assay = "qvalue", alpha = 0.1, ...) 8 | } 9 | \arguments{ 10 | \item{object}{A \code{\link{SummarizedBenchmark}} object.} 11 | 12 | \item{assay}{The name of an assay.} 13 | 14 | \item{alpha}{An alpha value.} 15 | 16 | \item{...}{Further arguments passed to \code{\link{upset}}} 17 | } 18 | \value{ 19 | An upseR plot. 20 | } 21 | \description{ 22 | This function looks for an assay, called by default 'qvalue', and given an alpha threshold, 23 | it binarizes the assay matrix depending on whether its values are below the alpha 24 | threshold. Then it uses the function \code{\link{upset}} to plot the overlaps. 25 | The plot is only generated if at least 2 methods have observations that pass 26 | the alpha threshold. 27 | } 28 | \examples{ 29 | data(sb) 30 | \dontrun{ 31 | plotMethodsOverlap(sb) 32 | } 33 | 34 | } 35 | \seealso{ 36 | \code{\link{plotROC}}, \code{\link{estimatePerformanceMetrics}} 37 | } 38 | \author{ 39 | Alejandro Reyes 40 | } 41 | -------------------------------------------------------------------------------- /man/printMethod.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/printMethod.R 3 | \name{printMethod} 4 | \alias{printMethod} 5 | \alias{printMethods} 6 | \title{Pretty print methods in a BenchDesign object} 7 | \usage{ 8 | printMethod(bd, n = NULL) 9 | 10 | printMethods(bd) 11 | } 12 | \arguments{ 13 | \item{bd}{BenchDesign object.} 14 | 15 | \item{n}{name of a method in the BenchDesign to show.} 16 | } 17 | \value{ 18 | Brief description is returned to console. 19 | } 20 | \description{ 21 | Print out details about a method included in 22 | the BenchDesign. The \code{printMethods} function is just a wrapper 23 | to call \code{printMethod} on all methods in the BenchDesign. 24 | } 25 | \examples{ 26 | ## create empty BenchDesign 27 | bench <- BenchDesign() 28 | 29 | ## currently no methods 30 | printMethods(bench) 31 | 32 | ## add method 33 | bench <- addMethod(bench, label = "method_a", p.adjust) 34 | bench <- addMethod(bench, label = "method_b", qvalue::qvalue) 35 | 36 | ## show a single method 37 | printMethod(bench, "method_a") 38 | 39 | ## show all methods 40 | printMethods(bench) 41 | 42 | } 43 | \seealso{ 44 | \code{\link{BDMethod-class}}, \code{\link{BenchDesign-class}} 45 | } 46 | \author{ 47 | Patrick Kimes 48 | } 49 | -------------------------------------------------------------------------------- /R/BDData-show.R: -------------------------------------------------------------------------------- 1 | .show.BDData <- function(object) { 2 | cat(stringr::str_pad("BenchDesign Data (BDData) ", 60, "right", pad = "-"), "\n") 3 | dtype <- object@type 4 | cat(" type:", object@type, "\n") 5 | if (dtype == "data") { 6 | if (is(object@data, "data.frame")) { 7 | cat(" data:\n") 8 | desc <- as.data.frame(object@data) 9 | } else { 10 | cat(" data summary:\n") 11 | desc <- data.frame(name = seq_len(length(object@data)), 12 | class = unlist(sapply(object@data, class)), 13 | length = unlist(sapply(object@data, length))) 14 | desc$name <- names(object@data) 15 | row.names(desc) <- NULL 16 | } 17 | print(utils::head(desc, n = 10)) 18 | if (nrow(desc) > 10) 19 | cat(" ... with", nrow(desc) - 10, "more rows.\n") 20 | } 21 | if (dtype == "md5hash") 22 | cat(" MD5 hash:", stringr::str_trunc(object@data, 50), "\n") 23 | } 24 | 25 | #' Show BDData object 26 | #' 27 | #' @param object BDData object to show 28 | #' 29 | #' @return 30 | #' Print description of BDData object to console 31 | #' 32 | #' @importFrom stringr str_pad 33 | #' @importFrom utils head 34 | #' @rdname BDData-show 35 | setMethod("show", signature(object = "BDData"), .show.BDData) 36 | -------------------------------------------------------------------------------- /R/BDMethodList-show.R: -------------------------------------------------------------------------------- 1 | .show.BDMethodList <- function(object) { 2 | cat(stringr::str_pad("BenchDesign Method List (BDMethodList) ", 60, "right", "-"), "\n") 3 | cat("list of", length(object), ifelse(length(object) == 1L, "method", "methods"), "\n") 4 | if (length(object) < 1) { 5 | cat(" none\n") 6 | } else { 7 | max_c <- 20 8 | m1 <- max(nchar(names(utils::head(object)))) 9 | m1 <- min(m1, max_c) 10 | m2 <- max(nchar(sapply(utils::head(object), function(x) { rlang::quo_text(x@fc) }))) 11 | m2 <- min(m2, max_c) 12 | for (n in names(object)) { 13 | p1 <- stringr::str_pad(stringr::str_trunc(n, max_c), m1 + 1, "left", " ") 14 | p2 <- stringr::str_pad(stringr::str_trunc(gsub("\n", ";", rlang::quo_text(object[[n]]@fc)), max_c), 15 | m2 + 1, "right", " ") 16 | cat(" method: ", p1, "; func: ", p2, "\n", sep="") 17 | } 18 | } 19 | } 20 | 21 | #' Show BDMethodList object 22 | #' 23 | #' @param object BDMethodList object to show 24 | #' 25 | #' @return 26 | #' Print description of BDMethodList object to console 27 | #' 28 | #' @importFrom stringr str_pad str_trunc 29 | #' @importFrom rlang quo_text 30 | #' @importFrom utils head 31 | #' @rdname BDMethodList-show 32 | setMethod("show", signature(object = "BDMethodList"), .show.BDMethodList) 33 | -------------------------------------------------------------------------------- /man/dropMethod.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dropMethod.R 3 | \name{dropMethod} 4 | \alias{dropMethod} 5 | \title{Remove method from BenchDesign object} 6 | \usage{ 7 | dropMethod(bd, label) 8 | } 9 | \arguments{ 10 | \item{bd}{\code{\link[=BenchDesign-class]{BenchDesign}} object.} 11 | 12 | \item{label}{Character name of method.} 13 | } 14 | \value{ 15 | Modified \code{\link[=BenchDesign-class]{BenchDesign}} object with 16 | specified method dropped. 17 | } 18 | \description{ 19 | Takes a \code{\link[=BenchDesign-class]{BenchDesign}} object and the name of an 20 | existing method 21 | and returns a reduced \code{\link[=BenchDesign-class]{BenchDesign}} object with the 22 | method removed. 23 | } 24 | \examples{ 25 | ## empty BenchDesign 26 | bench <- BenchDesign() 27 | 28 | ## add methods 29 | bench <- addMethod(bench, label = "bonf", func = p.adjust, 30 | params = rlang::quos(p = pval, method = "bonferroni")) 31 | bench <- addMethod(bench, label = "BH", func = p.adjust, 32 | params = rlang::quos(p = pval, method = "BH")) 33 | BDMethodList(bench) 34 | 35 | ## remove methods 36 | bench <- dropMethod(bench, label = "bonf") 37 | BDMethodList(bench) 38 | 39 | } 40 | \seealso{ 41 | \code{\link{modifyMethod}}, \code{\link{expandMethod}}, \code{\link{addMethod}} 42 | } 43 | \author{ 44 | Patrick Kimes 45 | } 46 | -------------------------------------------------------------------------------- /man/performanceMetrics-setter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, 3 | % R/SummarizedBenckmark-accessors.R 4 | \docType{methods} 5 | \name{performanceMetrics<-} 6 | \alias{performanceMetrics<-} 7 | \alias{performanceMetrics<-,SummarizedBenchmark,SimpleList-method} 8 | \title{Set performance metrics in SummarizedBenchmark object} 9 | \usage{ 10 | performanceMetrics(object, ...) <- value 11 | 12 | 13 | \S4method{performanceMetrics}{SummarizedBenchmark,SimpleList}(object) <- value 14 | } 15 | \arguments{ 16 | \item{object}{a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object.} 17 | 18 | \item{...}{futher arguments, perhaps used by methods.} 19 | 20 | \item{value}{a SimpleList of the same length as the number of assays.} 21 | } 22 | \value{ 23 | Silently, the newly specified SimpleList of performance metric lists. 24 | } 25 | \description{ 26 | Replaces the list of performance metrics in a 27 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object with 28 | a new list of performance metric lists. 29 | } 30 | \examples{ 31 | data(sb) 32 | performanceMetrics(sb) 33 | performanceMetrics(sb) <- SimpleList(qvalue = list(), logFC = list()) 34 | 35 | } 36 | \seealso{ 37 | \code{\link{addPerformanceMetric}}, \code{\link{estimatePerformanceMetrics}}, \code{\link{performanceMetrics}} 38 | } 39 | \author{ 40 | Alejandro Reyes 41 | } 42 | -------------------------------------------------------------------------------- /man/performanceMetrics.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, 3 | % R/SummarizedBenckmark-accessors.R 4 | \docType{methods} 5 | \name{performanceMetrics} 6 | \alias{performanceMetrics} 7 | \alias{performanceMetrics,SummarizedBenchmark-method} 8 | \title{Get performance metrics in SummarizedBenchmark object} 9 | \usage{ 10 | performanceMetrics(object, ...) 11 | 12 | \S4method{performanceMetrics}{SummarizedBenchmark}(object, assay = NULL) 13 | } 14 | \arguments{ 15 | \item{object}{a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object.} 16 | 17 | \item{...}{futher arguments, perhaps used by methods.} 18 | 19 | \item{assay}{a character string indicating an assay name.} 20 | } 21 | \value{ 22 | A SimpleList with one element for each assay. Each element of the list contains a list of performance metric functions. 23 | } 24 | \description{ 25 | Given a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object, 26 | returns a list of lists of performance metrics that have been defined for 27 | each assay. Optionally, if \code{assay =} is specified, performance metrics 28 | for only the specified subset of specified assays are returned. 29 | } 30 | \examples{ 31 | data(sb) 32 | performanceMetrics(sb) 33 | performanceMetrics(sb, assay = "qvalue") 34 | 35 | } 36 | \seealso{ 37 | \code{\link{addPerformanceMetric}}, \code{\link{estimatePerformanceMetrics}} 38 | } 39 | \author{ 40 | Alejandro Reyes 41 | } 42 | -------------------------------------------------------------------------------- /man/BenchDesign-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllClasses.R 3 | \docType{class} 4 | \name{BenchDesign-class} 5 | \alias{BenchDesign-class} 6 | \title{BenchDesign class} 7 | \description{ 8 | Along with the SummarizedBenchmark class, one of the two main 9 | classses of the SummarizedBenchmark package. The BenchDesign class serves as 10 | a container for both the set of methods to be benchmarked and optionally the 11 | data to be used for benchmarking. 12 | 13 | Methods are organized as BDMethod objects and stored in as a list using the 14 | BDMethodList class. The BDData class is used to store benchmarking data, or 15 | in some cases, just the MD5 hash of the original data set. Any list object, 16 | including data.frame objects, can be specified for data. More details on the 17 | component classes are provided in the corresponding class documentation. 18 | 19 | For details on how to create new BenchDesign objects, see the 20 | documentation for the \code{\link{BenchDesign}} constructor. 21 | } 22 | \section{Slots}{ 23 | 24 | \describe{ 25 | \item{\code{data}}{a list containing the data to be used in the benchmark.} 26 | 27 | \item{\code{methods}}{a \code{BDMethodList} list of \code{BDMethod} objects to be 28 | compared in the benchmark.} 29 | }} 30 | 31 | \seealso{ 32 | \code{\link{BenchDesign}}, \code{\link{BDMethod-class}}, \code{\link{BDMethodList-class}}, \code{\link{BDData-class}} 33 | } 34 | \author{ 35 | Patrick Kimes 36 | } 37 | -------------------------------------------------------------------------------- /R/printMethod.R: -------------------------------------------------------------------------------- 1 | #' Pretty print methods in a BenchDesign object 2 | #' 3 | #' Print out details about a method included in 4 | #' the BenchDesign. The `printMethods` function is just a wrapper 5 | #' to call `printMethod` on all methods in the BenchDesign. 6 | #' 7 | #' @param bd BenchDesign object. 8 | #' @param n name of a method in the BenchDesign to show. 9 | #' 10 | #' @return 11 | #' Brief description is returned to console. 12 | #' 13 | #' @examples 14 | #' ## create empty BenchDesign 15 | #' bench <- BenchDesign() 16 | #' 17 | #' ## currently no methods 18 | #' printMethods(bench) 19 | #' 20 | #' ## add method 21 | #' bench <- addMethod(bench, label = "method_a", p.adjust) 22 | #' bench <- addMethod(bench, label = "method_b", qvalue::qvalue) 23 | #' 24 | #' ## show a single method 25 | #' printMethod(bench, "method_a") 26 | #' 27 | #' ## show all methods 28 | #' printMethods(bench) 29 | #' 30 | #' @seealso \code{\link{BDMethod-class}}, \code{\link{BenchDesign-class}} 31 | #' @md 32 | #' @importFrom stringr str_pad 33 | #' @export 34 | #' @author Patrick Kimes 35 | printMethod <- function(bd, n = NULL) { 36 | if (is.null(n)) 37 | n <- names(bd@methods)[1] 38 | stopifnot(n %in% names(bd@methods)) 39 | 40 | cat(stringr::str_pad(paste0(n, " "), 60, "right", pad = "-"), "\n") 41 | show(bd@methods[[n]]) 42 | } 43 | 44 | 45 | #' @rdname printMethod 46 | #' @export 47 | printMethods <- function(bd) { 48 | for (n in names(bd@methods)) { 49 | printMethod(bd, n) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /inst/NEWS.Rd: -------------------------------------------------------------------------------- 1 | \name{sbnews} 2 | \title{SummarizedBenchmark News} 3 | \encoding{UTF-8} 4 | 5 | \section{Changes in version 2.3.7 (2019-09-18)}{ 6 | \itemize{ 7 | \item Bug fix: update call on `tidyr::gather` in `tidyUpMetrics` 8 | to work after bump in `tidyr` version to 1.0.0 on CRAN 9 | } 10 | } 11 | 12 | \section{Changes in version 2.3.6 (2019-09-06)}{ 13 | \itemize{ 14 | \item Replace `sc_example_counts` dataset from `scater` with call to 15 | `scater::mockSCE` 16 | \item Fix error thrown by `show.BDData` when dataset is a list 17 | } 18 | } 19 | 20 | \section{Changes in version 2.3.5 (2019-06-22)}{ 21 | \itemize{ 22 | \item Fix index title for "Feature: Error Handling" vignette 23 | \item Replace Rd references to `rlang::quos` with `rlang:quotation` 24 | topic page since aliasing appears to be causing warnings on 25 | Windows build 26 | \item Update pkgdown site with latest devel docs 27 | } 28 | } 29 | 30 | \section{Changes in version 2.3.4 (2019-06-18)}{ 31 | \itemize{ 32 | \item Fix scRNAseq simulation case study vignette to work with 33 | changes in `scRNAseq` package 34 | \item Add code examples and import statements to pass BiocCheck 35 | \item Update pkgdown site with latest devel docs 36 | } 37 | } 38 | 39 | \section{Changes in version 2.3.3 (2019-06-14)}{ 40 | \itemize{ 41 | \item Added NEWS file to track changes 42 | \item Incorporated major updates to package documentation and vignettes 43 | \item Added pkgdown site 44 | } 45 | } 46 | 47 | -------------------------------------------------------------------------------- /R/dropMethod.R: -------------------------------------------------------------------------------- 1 | #' Remove method from BenchDesign object 2 | #' 3 | #' @description 4 | #' Takes a \code{\link[=BenchDesign-class]{BenchDesign}} object and the name of an 5 | #' existing method 6 | #' and returns a reduced \code{\link[=BenchDesign-class]{BenchDesign}} object with the 7 | #' method removed. 8 | #' 9 | #' @param bd \code{\link[=BenchDesign-class]{BenchDesign}} object. 10 | #' @param label Character name of method. 11 | #' 12 | #' @return 13 | #' Modified \code{\link[=BenchDesign-class]{BenchDesign}} object with 14 | #' specified method dropped. 15 | #' 16 | #' @examples 17 | #' ## empty BenchDesign 18 | #' bench <- BenchDesign() 19 | #' 20 | #' ## add methods 21 | #' bench <- addMethod(bench, label = "bonf", func = p.adjust, 22 | #' params = rlang::quos(p = pval, method = "bonferroni")) 23 | #' bench <- addMethod(bench, label = "BH", func = p.adjust, 24 | #' params = rlang::quos(p = pval, method = "BH")) 25 | #' BDMethodList(bench) 26 | #' 27 | #' ## remove methods 28 | #' bench <- dropMethod(bench, label = "bonf") 29 | #' BDMethodList(bench) 30 | #' 31 | #' @seealso \code{\link{modifyMethod}}, \code{\link{expandMethod}}, \code{\link{addMethod}} 32 | #' @md 33 | #' @export 34 | #' @author Patrick Kimes 35 | dropMethod <- function(bd, label) { 36 | UseMethod("dropMethod") 37 | } 38 | 39 | #' @export 40 | dropMethod.BenchDesign <- function(bd, label) { 41 | ## verify that method definition already exists 42 | if(!(label %in% names(bd@methods))) { 43 | stop("Specified method is not defined in BenchDesign.") 44 | } 45 | 46 | bd@methods[[label]] <- NULL 47 | return(bd) 48 | } 49 | 50 | -------------------------------------------------------------------------------- /tests/testthat/test_addPerformanceFunction.R: -------------------------------------------------------------------------------- 1 | context("performanceFunctions") 2 | test_that("test='addPerformanceMetrics' returns correct error messages", { 3 | data(sb) 4 | fun <- function(query, truth, alpha=0.1){ 5 | goodHits <- sum( (query < alpha) & truth == 1 ) 6 | goodHits / sum(truth == 1) 7 | } 8 | expect_error( addPerformanceMetric( 9 | object=1:10, 10 | assay="qvalue", 11 | evalMetric="TPR", 12 | evalFunction=fun) ) 13 | expect_error( addPerformanceMetric( 14 | sb, 15 | assay="notexisting", 16 | evalMetric="TPR", 17 | evalFunction=fun) ) 18 | expect_error( addPerformanceMetric( 19 | sb, 20 | assay="notexisting", 21 | evalMetric=NULL, 22 | evalFunction=fun) ) 23 | expect_error( addPerformanceMetric( 24 | sb, 25 | assay=NULL, 26 | evalMetric="TPR", 27 | evalFunction=fun) ) 28 | expect_error( addPerformanceMetric( 29 | sb, 30 | assay="notexisting", 31 | evalMetric=c("TPR", "FDR"), 32 | evalFunction=fun) ) 33 | expect_error( addPerformanceMetric( 34 | sb, 35 | assay="qvalue", 36 | evalFunction=fun) ) 37 | fun <- function(query, truth2, alpha=0.1){ 38 | goodHits <- sum( (query < alpha) & truth == 1 ) 39 | goodHits / sum(truth == 1) 40 | } 41 | expect_error( addPerformanceMetric( 42 | sb, 43 | assay="qvalue", 44 | evalMetric="TPR", 45 | evalFunction=fun ) ) 46 | fun <- function(query, truth, alpha){ 47 | goodHits <- sum( (query < alpha) & truth == 1 ) 48 | goodHits / sum(truth == 1) 49 | } 50 | expect_error( addPerformanceMetric( 51 | sb, 52 | assay="qvalue", 53 | evalMetric="TPR", 54 | evalFunction=fun ) ) 55 | } ) 56 | -------------------------------------------------------------------------------- /man/BDMethod-setter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/BDMethodList-accessors.R, 3 | % R/BenchDesign-accessors.R 4 | \docType{methods} 5 | \name{BDMethod<-} 6 | \alias{BDMethod<-} 7 | \alias{BDMethod<-,BDMethodList,character,BDMethod-method} 8 | \alias{BDMethod<-,BDMethodList,character,NULL-method} 9 | \alias{BDMethod<-,BenchDesign,character,BDMethod-method} 10 | \alias{BDMethod<-,BenchDesign,character,NULL-method} 11 | \title{Set method in list or BenchDesign object} 12 | \usage{ 13 | BDMethod(x, i) <- value 14 | 15 | \S4method{BDMethod}{BDMethodList,character,BDMethod}(x, i) <- value 16 | 17 | \S4method{BDMethod}{BDMethodList,character,`NULL`}(x, i) <- value 18 | 19 | \S4method{BDMethod}{BenchDesign,character,BDMethod}(x, i) <- value 20 | 21 | \S4method{BDMethod}{BenchDesign,character,`NULL`}(x, i) <- value 22 | } 23 | \arguments{ 24 | \item{x}{\code{\link[=BenchDesign-class]{BenchDesign}} or 25 | \code{\link[=BDMethodList-class]{BDMethodList}} object.} 26 | 27 | \item{i}{character name of method.} 28 | 29 | \item{value}{\code{\link[=BDMethod-class]{BDMethod}} or \code{NULL}.} 30 | } 31 | \value{ 32 | modified BenchDesign object 33 | } 34 | \description{ 35 | Adds, replaces or removes a named \code{\link[=BDMethod-class]{BDMethod}} method 36 | in a \code{\link[=BDMethodList-class]{BDMethodList}} or 37 | \code{\link[=BenchDesign-class]{BenchDesign}} object with a specified 38 | \code{\link[=BDMethod-class]{BDMethod}} object. 39 | 40 | An existing method can be removed by setting the value to \code{NULL}. 41 | } 42 | \examples{ 43 | bd <- BenchDesign() 44 | BDMethod(bd, "avg") <- BDMethod(x = base::mean) 45 | bd 46 | 47 | } 48 | \seealso{ 49 | \code{\link{BDMethod}} 50 | } 51 | \author{ 52 | Patrick Kimes 53 | } 54 | -------------------------------------------------------------------------------- /R/BenchDesign-accessors.R: -------------------------------------------------------------------------------- 1 | #' @rdname BDData-setter 2 | #' @exportMethod "BDData<-" 3 | setReplaceMethod("BDData", 4 | signature(x = "BenchDesign", value = "BDDataOrNULL"), 5 | function (x, value) { 6 | x@data <- value 7 | x 8 | }) 9 | 10 | 11 | #' @rdname BDMethod-setter 12 | #' @exportMethod "BDMethod<-" 13 | setReplaceMethod("BDMethod", 14 | signature(x = "BenchDesign", i = "character", value = "BDMethod"), 15 | function (x, i, value) { 16 | x@methods[[i]] <- value 17 | x 18 | }) 19 | 20 | #' @rdname BDMethod-setter 21 | #' @exportMethod "BDMethod<-" 22 | setReplaceMethod("BDMethod", 23 | signature(x = "BenchDesign", i = "character", value = "NULL"), 24 | function (x, i, value) { 25 | x@methods[[i]] <- value 26 | x 27 | }) 28 | 29 | 30 | #' @rdname BDMethodList-setter 31 | #' @exportMethod "BDMethodList<-" 32 | setReplaceMethod("BDMethodList", 33 | signature(x = "BenchDesign", value = "BDMethodList"), 34 | function (x, value) { 35 | x@methods <- value 36 | x 37 | }) 38 | 39 | 40 | #' @rdname BDMethod 41 | #' @exportMethod "BDMethod" 42 | setMethod("BDMethod", 43 | signature(x = "BenchDesign"), 44 | function(x, i = 1) { 45 | BDMethod(x@methods, i = i) 46 | }) 47 | 48 | #' @rdname BDMethodList 49 | #' @exportMethod "BDMethodList" 50 | setMethod("BDMethodList", 51 | signature(x = "BenchDesign"), 52 | function(..., x) { 53 | x@methods 54 | }) 55 | -------------------------------------------------------------------------------- /man/addPerformanceMetric.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/PerformanceMetrics.R 3 | \name{addPerformanceMetric} 4 | \alias{addPerformanceMetric} 5 | \title{Add performance metric to SummarizedBenchmark object} 6 | \usage{ 7 | addPerformanceMetric(object, evalMetric, assay, evalFunction = NULL) 8 | } 9 | \arguments{ 10 | \item{object}{A \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object.} 11 | 12 | \item{evalMetric}{A string with the name of the evaluation metric.} 13 | 14 | \item{assay}{A string with an assay name. Indicates the assay that should be 15 | given as input to this performance metric.} 16 | 17 | \item{evalFunction}{A function that calculates a performance metric. It should 18 | contain at least two arguments, query and truth, where query is the output vector 19 | of a method and truth is the vector of true values. If additional parameters 20 | are specified, they must contain default values. If NULL, the 'evalMetric' string must 21 | be the name of a predefined metric available through 'availableMetrics()$function'.} 22 | } 23 | \value{ 24 | A \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. 25 | } 26 | \description{ 27 | This is a function to define performance metrics for benchmarking methods. 28 | The function is saved into the \code{performanceMetrics} slot. 29 | } 30 | \examples{ 31 | data( sb ) 32 | sb <- addPerformanceMetric( 33 | object=sb, 34 | assay="qvalue", 35 | evalMetric="TPR", 36 | evalFunction = function( query, truth, alpha=0.1 ){ 37 | goodHits <- sum( (query < alpha) & truth == 1 ) 38 | goodHits / sum(truth == 1) 39 | } 40 | ) 41 | 42 | } 43 | \seealso{ 44 | \code{\link{availableMetrics}}, \code{\link{performanceMetrics}} 45 | } 46 | \author{ 47 | Alejandro Reyes 48 | } 49 | -------------------------------------------------------------------------------- /man/SummarizedBenchmark-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllClasses.R 3 | \docType{class} 4 | \name{SummarizedBenchmark-class} 5 | \alias{SummarizedBenchmark-class} 6 | \title{SummarizedBenchmark class} 7 | \description{ 8 | Extension of the \code{\link{RangedSummarizedExperiment}} to 9 | store the output of different methods intended for the same purpose 10 | in a given dataset. For example, a differential expression analysis could be 11 | done using \pkg{limma}-voom, \pkg{edgeR} and \pkg{DESeq2}. The 12 | SummarizedBenchmark class provides a framework that is useful to store, benckmark and 13 | compare results. 14 | } 15 | \section{Slots}{ 16 | 17 | \describe{ 18 | \item{\code{performanceMetrics}}{A \code{\link{SimpleList}} of the same length 19 | as the number of \code{\link{assays}} containing performance 20 | functions to be compared with the ground truths.} 21 | 22 | \item{\code{BenchDesign}}{A \code{\link{BenchDesign}} originally used to generate the 23 | results in the object.} 24 | }} 25 | 26 | \examples{ 27 | ## loading the example data from iCOBRA 28 | library(iCOBRA) 29 | data(cobradata_example) 30 | 31 | ## a bit of data wrangling and reformatting 32 | assays <- list( 33 | qvalue=cobradata_example@padj, 34 | logFC=cobradata_example@score ) 35 | assays[["qvalue"]]$DESeq2 <- p.adjust(cobradata_example@pval$DESeq2, method="BH") 36 | groundTruth <- DataFrame( cobradata_example@truth[,c("status", "logFC")] ) 37 | colnames(groundTruth) <- names( assays ) 38 | colData <- DataFrame( method=colnames(assays[[1]]) ) 39 | groundTruth <- groundTruth[rownames(assays[[1]]),] 40 | 41 | ## constructing a SummarizedBenchmark object 42 | sb <- SummarizedBenchmark( 43 | assays=assays, colData=colData, 44 | groundTruth=groundTruth ) 45 | 46 | } 47 | \author{ 48 | Alejandro Reyes 49 | } 50 | -------------------------------------------------------------------------------- /tests/testthat/test_metricsRerun.R: -------------------------------------------------------------------------------- 1 | context("estimatePeformanceMetrics") 2 | test_that("test='estimatePerformanceMetrics' with rerun=FALSE works appropiatetly", { 3 | df <- data.frame(pval = rnorm(100)) 4 | bench <- BenchDesign(data = df) 5 | ## add methods 6 | bench <- addMethod(bench, label = "bonf", func = p.adjust, 7 | params = rlang::quos(p = pval, method = "bonferroni")) 8 | bench <- addMethod(bench, label = "BH", func = p.adjust, 9 | params = rlang::quos(p = pval, method = "BH")) 10 | ## evaluate benchmark experiment 11 | sb <- buildBench(bench) 12 | ## evaluate benchmark experiment w/ data sepecified 13 | sb <- buildBench(bench, data = df) 14 | sb <- addPerformanceMetric( sb, assay="default", evalMetric="rejections" ) 15 | 16 | ## calling it twice throws an error saying it is up to date ## 17 | expect_error( estimatePerformanceMetrics( 18 | estimatePerformanceMetrics(sb, addColData=TRUE, rerun=FALSE ), 19 | addColData=TRUE, rerun=FALSE ) ) 20 | 21 | ## results are identical between rerun=TRUE and rerun=FALSE ## 22 | sb <- estimatePerformanceMetrics(sb, addColData=TRUE) 23 | bench <- addMethod( bench, label="BH2", func=p.adjust, 24 | params = rlang::quos(p = pval, method = "BH") ) 25 | sb2 <- updateBench( sb, bd=bench, dryrun=FALSE ) 26 | res1 <- colData( estimatePerformanceMetrics( sb2, rerun=FALSE, addColData=TRUE ) ) 27 | sb2 <- estimatePerformanceMetrics( sb2, rerun=TRUE, addColData=TRUE ) 28 | res2 <- colData( sb2 ) 29 | expect_identical( res2[,colnames( res1 )], res1 ) 30 | 31 | ## Also works when modifying a method ## 32 | bench <- modifyMethod( bench, 33 | label="BH", 34 | param=rlang::quos(p=pval, method="hommel")) 35 | sb2 <- updateBench( sb2, bd=bench, dryrun=FALSE ) 36 | expect_s4_class( estimatePerformanceMetrics( sb2, rerun=FALSE ), "DataFrame") 37 | }) 38 | -------------------------------------------------------------------------------- /man/tidyBDMethod.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/BDMethod-tidy.R 3 | \docType{methods} 4 | \name{tidyBDMethod} 5 | \alias{tidyBDMethod} 6 | \alias{tidyBDMethod,BDMethod-method} 7 | \alias{tidyBDMethod,list-method} 8 | \alias{tidyBDMethod,SimpleList-method} 9 | \alias{tidyBDMethod,BenchDesign-method} 10 | \title{Tidy BDMethod Data} 11 | \usage{ 12 | tidyBDMethod(obj, dat = NULL, eval = FALSE, label = FALSE) 13 | 14 | \S4method{tidyBDMethod}{BDMethod}(obj, dat, eval) 15 | 16 | \S4method{tidyBDMethod}{list}(obj, dat = NULL, eval = FALSE, 17 | label = FALSE) 18 | 19 | \S4method{tidyBDMethod}{SimpleList}(obj, dat = NULL, eval = FALSE, 20 | label = FALSE) 21 | 22 | \S4method{tidyBDMethod}{BenchDesign}(obj, dat = NULL, eval = FALSE, 23 | label = FALSE) 24 | } 25 | \arguments{ 26 | \item{obj}{BDMethod object, list/List of BDMethod objects (e.g. a 27 | BDMethodList), or a BenchDesign object} 28 | 29 | \item{dat}{optional data object to use when evaluating any 30 | unevaluated expressionsin \code{bdm} meta data. 31 | (default = NULL)} 32 | 33 | \item{eval}{logical whether to evaluate any quosures in the meta slot 34 | of the BDMethod objects. (default = FALSE)} 35 | 36 | \item{label}{logical whether to add a "label" column to the resulting 37 | table containing the names of the methods if \code{obj} was 38 | specified as a named list. (default = FALSE)} 39 | } 40 | \value{ 41 | A named vector of meta data if only a single BDMethod object specified, else 42 | a tibble of meta data for the specified list of methods. 43 | } 44 | \description{ 45 | A helper function to extract information for a single 46 | or multiple BDMethod object or a list of BDMethod objects. 47 | } 48 | \details{ 49 | If any quosures are specified to the "meta" slot of a BDMethod object, 50 | the quosure is converted to a text string using \code{rlang::quo_text}. 51 | } 52 | \author{ 53 | Patrick Kimes 54 | } 55 | -------------------------------------------------------------------------------- /R/BenchDesign-show.R: -------------------------------------------------------------------------------- 1 | .show.BenchDesign <- function(object) { 2 | cat(stringr::str_pad("BenchDesign ", 60, "right", "-"), "\n") 3 | cat(" benchmark data:\n") 4 | if (!is.null(object@data)) { 5 | dtype <- object@data@type 6 | cat(" type:", object@data@type, "\n") 7 | if (dtype == "data") 8 | cat(" names:", paste(names(object@data@data), collapse=", "), "\n") 9 | if (dtype == "md5hash") 10 | cat(" MD5 hash:", stringr::str_trunc(object@data@data, 50), "\n") 11 | } else { 12 | cat(" NULL\n") 13 | } 14 | cat(" benchmark methods:\n") 15 | if (length(object@methods) < 1) { 16 | cat(" none\n") 17 | } else { 18 | max_c <- 20 19 | m1 <- max(nchar(names(utils::head(object@methods)))) 20 | m1 <- min(m1, max_c) 21 | m2 <- max(nchar(sapply(utils::head(object@methods), function(x) { rlang::quo_text(x@fc) }))) 22 | m2 <- min(m2, max_c) 23 | for (n in names(utils::head(object@methods, 5))) { 24 | p1 <- stringr::str_pad(stringr::str_trunc(n, max_c), m1 + 1, "left", " ") 25 | p2 <- stringr::str_pad(stringr::str_trunc(gsub("\n", ";", rlang::quo_text(object@methods[[n]]@fc)), max_c), 26 | m2 + 1, "right", " ") 27 | cat(" method: ", p1, "; func: ", p2, "\n", sep="") 28 | } 29 | if (length(object@methods) > 5 ) { 30 | cat(" ... and", length(object@methods) - 5, "more methods.\n") 31 | } 32 | } 33 | } 34 | 35 | #' Show BenchDesign object 36 | #' 37 | #' @param object BenchDesign object to show 38 | #' 39 | #' @return 40 | #' Print description of BenchDesign object to console 41 | #' 42 | #' @importFrom stringr str_pad str_trunc 43 | #' @importFrom rlang quo_text 44 | #' @importFrom utils head 45 | #' @rdname BenchDesign-show 46 | setMethod("show", signature(object = "BenchDesign"), .show.BenchDesign) 47 | -------------------------------------------------------------------------------- /R/BDMethod-show.R: -------------------------------------------------------------------------------- 1 | .show.BDMethod <- function(object) { 2 | cat(stringr::str_pad("BenchDesign Method (BDMethod) ", 60, "right", pad = "-"), "\n") 3 | cat(" method: ") 4 | func_q <- rlang::quo_text(object@fc, nlines = 3) 5 | if (!grepl("^function\\(", func_q)) { 6 | cat(stringr::str_trunc(func_q, 50)) 7 | func_q <- rlang::quo_text(object@f, nlines = 3) 8 | } 9 | func_q <- gsub("\\s*\n\\s*", " ", func_q) 10 | cat("\n ", stringr::str_trunc(func_q, 56), "\n") 11 | cat(" parameters:\n") 12 | if (length(object@params) > 0) { 13 | param_n <- names(object@params) 14 | param_q <- sapply(object@params, quo_text) 15 | for (i in seq(param_n)) 16 | cat(stringr::str_trunc(paste(" ", param_n[i], ":", param_q[i]), 60), "\n") 17 | } else { 18 | cat(" none\n") 19 | } 20 | cat(" post:\n") 21 | if (!is.null(object@post)) { 22 | post_n <- names(object@post) 23 | post_q <- lapply(object@post, rlang::quo_text, nlines = 3) 24 | post_q <- gsub("\\s*\n\\s*", " ", post_q) 25 | for (i in seq(post_n)) 26 | cat(stringr::str_trunc(paste(" ", post_n[i], ":", post_q[i]), 60), "\n") 27 | } else { 28 | cat(" none\n") 29 | } 30 | cat(" meta:\n") 31 | if (!is.null(object@meta)) { 32 | meta_n <- names(object@meta) 33 | meta_q <- sapply(object@meta, quo_text) 34 | for (i in seq(meta_n)) 35 | cat(stringr::str_trunc(paste(" ", meta_n[i], ":", meta_q[i]), 60), "\n") 36 | } else { 37 | cat(" none\n") 38 | } 39 | } 40 | 41 | #' Show BDMethod object 42 | #' 43 | #' @param object BDMethod object to show 44 | #' 45 | #' @return 46 | #' Print description of BDMethod object to console 47 | #' 48 | #' @importFrom stringr str_pad str_trunc 49 | #' @importFrom rlang quo_text 50 | #' @rdname BDMethod-show 51 | setMethod("show", signature(object = "BDMethod"), .show.BDMethod) 52 | -------------------------------------------------------------------------------- /man/BDData.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/BDData-constructor.R 3 | \docType{methods} 4 | \name{BDData} 5 | \alias{BDData} 6 | \alias{BDData,ANY-method} 7 | \alias{BDData,BenchDesign-method} 8 | \alias{BDData,SummarizedBenchmark-method} 9 | \alias{BDData,BDData-method} 10 | \title{Create a new BDData object} 11 | \usage{ 12 | BDData(data) 13 | 14 | \S4method{BDData}{ANY}(data) 15 | 16 | \S4method{BDData}{BenchDesign}(data) 17 | 18 | \S4method{BDData}{SummarizedBenchmark}(data) 19 | 20 | \S4method{BDData}{BDData}(data) 21 | } 22 | \arguments{ 23 | \item{data}{a list object of data or MD5 hash string} 24 | } 25 | \value{ 26 | BDData object 27 | } 28 | \description{ 29 | Initializes a new BDData object of benchmarking data. 30 | 31 | Data sets are stored as BDData objects within BenchDesign 32 | objects as well as SummarizedBenchmark objects. However, because 33 | data is directly specified to the BenchDesign constructor, there 34 | is usually no need to call the BDData constructor to create completely 35 | new data objects. 36 | 37 | The BDData constructor is most useful for extracting the data sets 38 | contained in BenchDesign objects as well as SummarizedBenchmark objects. 39 | By default, the BDData object stored in SummarizedBenchmark objects will 40 | be MD5 hashes rather than the complete original data set. 41 | \code{\link{compareBDData}} can be used to compare various forms of 42 | BDData, as shown in the examples below. 43 | } 44 | \examples{ 45 | ## construct from data.frame 46 | datadf <- data.frame(x = 1:5, y = runif(5)) 47 | bdd_df <- BDData(datadf) 48 | bdd_df 49 | 50 | ## construct from MD5 hash of data.frame 51 | bdd_md5 <- BDData(digest::digest(datadf)) 52 | bdd_md5 53 | 54 | ## compare two BDData objects 55 | compareBDData(bdd_df, bdd_md5) 56 | 57 | ## note that the data is the same, i.e. the MD5 hashes match, but the 58 | ## data types ("data" vs. "md5has") are different 59 | 60 | } 61 | \seealso{ 62 | \code{\link{BDData-class}}, \code{\link{BenchDesign}} 63 | } 64 | \author{ 65 | Patrick Kimes 66 | } 67 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: SummarizedBenchmark 2 | Type: Package 3 | Title: Classes and methods for performing benchmark comparisons 4 | Version: 2.11.1 5 | Authors@R: c(person("Alejandro", "Reyes", role = "aut", 6 | email = "alejandro.reyes.ds@gmail.com", 7 | comment = c(ORCID = "0000-0001-8717-6612")), 8 | person("Patrick", "Kimes", role = c("aut", "cre"), 9 | email = "patrick.kimes@gmail.com", 10 | comment = c(ORCID = "0000-0001-6819-9077"))) 11 | BugReports: https://github.com/areyesq89/SummarizedBenchmark/issues 12 | URL: https://github.com/areyesq89/SummarizedBenchmark, http://bioconductor.org/packages/SummarizedBenchmark/ 13 | Description: This package defines the BenchDesign and SummarizedBenchmark classes for building, 14 | executing, and evaluating benchmark experiments of computational methods. The SummarizedBenchmark 15 | class extends the RangedSummarizedExperiment object, and is designed to provide infrastructure 16 | to store and compare the results of applying different methods to a shared data set. 17 | This class provides an integrated interface to store metadata such as method parameters and software 18 | versions as well as ground truths (when these are available) and evaluation metrics. 19 | biocViews: Software, Infrastructure 20 | Depends: R (>= 3.6), 21 | tidyr, 22 | SummarizedExperiment, 23 | S4Vectors, 24 | BiocGenerics, 25 | methods, 26 | UpSetR, 27 | rlang, 28 | stringr, 29 | utils, 30 | BiocParallel, 31 | ggplot2, 32 | mclust, 33 | dplyr, 34 | digest, 35 | sessioninfo, 36 | crayon, 37 | tibble 38 | Suggests: iCOBRA, 39 | BiocStyle, 40 | rmarkdown, 41 | knitr, 42 | magrittr, 43 | IHW, 44 | qvalue, 45 | testthat, 46 | DESeq2, 47 | edgeR, 48 | limma, 49 | tximport, 50 | readr, 51 | scRNAseq, 52 | splatter, 53 | scater, 54 | rnaseqcomp, 55 | biomaRt 56 | License: GPL (>= 3) 57 | Encoding: UTF-8 58 | LazyData: true 59 | VignetteBuilder: knitr 60 | RoxygenNote: 6.1.1 61 | Config/testthat/edition: 3 62 | -------------------------------------------------------------------------------- /docs/docsearch.js: -------------------------------------------------------------------------------- 1 | $(function() { 2 | 3 | // register a handler to move the focus to the search bar 4 | // upon pressing shift + "/" (i.e. "?") 5 | $(document).on('keydown', function(e) { 6 | if (e.shiftKey && e.keyCode == 191) { 7 | e.preventDefault(); 8 | $("#search-input").focus(); 9 | } 10 | }); 11 | 12 | $(document).ready(function() { 13 | // do keyword highlighting 14 | /* modified from https://jsfiddle.net/julmot/bL6bb5oo/ */ 15 | var mark = function() { 16 | 17 | var referrer = document.URL ; 18 | var paramKey = "q" ; 19 | 20 | if (referrer.indexOf("?") !== -1) { 21 | var qs = referrer.substr(referrer.indexOf('?') + 1); 22 | var qs_noanchor = qs.split('#')[0]; 23 | var qsa = qs_noanchor.split('&'); 24 | var keyword = ""; 25 | 26 | for (var i = 0; i < qsa.length; i++) { 27 | var currentParam = qsa[i].split('='); 28 | 29 | if (currentParam.length !== 2) { 30 | continue; 31 | } 32 | 33 | if (currentParam[0] == paramKey) { 34 | keyword = decodeURIComponent(currentParam[1].replace(/\+/g, "%20")); 35 | } 36 | } 37 | 38 | if (keyword !== "") { 39 | $(".contents").unmark({ 40 | done: function() { 41 | $(".contents").mark(keyword); 42 | } 43 | }); 44 | } 45 | } 46 | }; 47 | 48 | mark(); 49 | }); 50 | }); 51 | 52 | /* Search term highlighting ------------------------------*/ 53 | 54 | function matchedWords(hit) { 55 | var words = []; 56 | 57 | var hierarchy = hit._highlightResult.hierarchy; 58 | // loop to fetch from lvl0, lvl1, etc. 59 | for (var idx in hierarchy) { 60 | words = words.concat(hierarchy[idx].matchedWords); 61 | } 62 | 63 | var content = hit._highlightResult.content; 64 | if (content) { 65 | words = words.concat(content.matchedWords); 66 | } 67 | 68 | // return unique words 69 | var words_uniq = [...new Set(words)]; 70 | return words_uniq; 71 | } 72 | 73 | function updateHitURL(hit) { 74 | 75 | var words = matchedWords(hit); 76 | var url = ""; 77 | 78 | if (hit.anchor) { 79 | url = hit.url_without_anchor + '?q=' + escape(words.join(" ")) + '#' + hit.anchor; 80 | } else { 81 | url = hit.url + '?q=' + escape(words.join(" ")); 82 | } 83 | 84 | return url; 85 | } 86 | -------------------------------------------------------------------------------- /man/BDMethodList.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/BDMethodList-constructor.R, 3 | % R/BenchDesign-accessors.R, R/SummarizedBenckmark-accessors.R 4 | \docType{methods} 5 | \name{BDMethodList} 6 | \alias{BDMethodList} 7 | \alias{BDMethodList,ANY-method} 8 | \alias{coerce} 9 | \alias{BDMethodList,BenchDesign-method} 10 | \alias{BDMethodList,SummarizedBenchmark-method} 11 | \title{Create a new BDMethodList object} 12 | \usage{ 13 | BDMethodList(..., x = NULL) 14 | 15 | \S4method{BDMethodList}{ANY}(..., x = NULL) 16 | 17 | \S4method{BDMethodList}{BenchDesign}(..., x = NULL) 18 | 19 | \S4method{BDMethodList}{SummarizedBenchmark}(..., x = NULL) 20 | } 21 | \arguments{ 22 | \item{...}{a named list of \code{\link[=BDMethod-class]{BDMethod}} objects} 23 | 24 | \item{x}{a \code{\link[=BenchDesign-class]{BenchDesign}} or 25 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object to extract 26 | the list of methods from. (default = NULL)} 27 | } 28 | \value{ 29 | BDMethodList object 30 | } 31 | \description{ 32 | Initializes a new SimpleList of BenchDesign method (\code{\link[=BDMethod-class]{BDMethod}}) objects. 33 | 34 | Similar to \code{\link[=BDMethod-class]{BDMethod}} objects, \code{\link[=BDMethodList-class]{BDMethodList}} 35 | typically do not need to be 36 | directly constructed. Because the list of methods is only useful as art of a 37 | \code{\link[=BenchDesign-class]{BenchDesign}} object, it is more common to simply manipulate the 38 | list of methods through calls to the corresponding \code{\link[=BenchDesign-class]{BenchDesign}}, e.g. 39 | \code{\link{addMethod}} to add a new method to the list. 40 | 41 | The constructor can also be used to access the \code{\link[=BDMethodList-class]{BDMethodList}} list 42 | of methods in a \code{\link[=BenchDesign-class]{BenchDesign}} object. 43 | } 44 | \examples{ 45 | ## construct an empty list 46 | bdml <- BDMethodList() 47 | 48 | ## construct a list with BDMethod objects 49 | bdml <- BDMethodList(m_method = BDMethod(base::mean), 50 | s_method = BDMethod(function(x) { x^2 })) 51 | bdml 52 | 53 | ## construct a BenchDesign with a BDMethodList 54 | bd <- BenchDesign(methods = bdml) 55 | 56 | ## access the BDMethodList in the BenchDesign 57 | BDMethodList(bd) 58 | 59 | } 60 | \seealso{ 61 | \code{\link{BDMethodList-class}}, \code{\link{BenchDesign}}, \code{\link{BDMethod}} 62 | } 63 | \author{ 64 | Patrick Kimes 65 | } 66 | -------------------------------------------------------------------------------- /man/modifyMethod.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/modifyMethod.R 3 | \name{modifyMethod} 4 | \alias{modifyMethod} 5 | \title{Modify method in BenchDesign object} 6 | \usage{ 7 | modifyMethod(bd, label, params, .overwrite = FALSE) 8 | } 9 | \arguments{ 10 | \item{bd}{\code{\link[=BenchDesign-class]{BenchDesign}} object.} 11 | 12 | \item{label}{Character name of method to be modified.} 13 | 14 | \item{params}{Named quosure list created using \code{\link[rlang:quotation]{quos}} of 15 | \code{parameter = value} paiars to replace in the method definition. 16 | The \code{post}, and \code{meta} parameters of the method can be 17 | modified using the special keywords, \code{bd.post}, and \code{bd.meta} 18 | (the prefix denoting that these values should modify \code{\link[=BenchDesign-class]{BenchDesign}} 19 | parameters). All other named parameters will be added to the list of 20 | parameters to be passed to \code{func}.} 21 | 22 | \item{.overwrite}{Logical whether to overwrite the complete existing list of 23 | parameters to be passed to \code{func} (\code{TRUE}), or to simply add 24 | the new parameters to the existing list and only replace overlapping 25 | parameters (\code{FALSE}). (default = \code{FALSE})} 26 | } 27 | \value{ 28 | Modified \code{\link[=BenchDesign-class]{BenchDesign}} object with single method 29 | parameters modified. 30 | } 31 | \description{ 32 | Takes a \code{\link[=BenchDesign-class]{BenchDesign}} object, the name of an 33 | existing method, and new parameter specifications, 34 | and returns a modified \code{\link[=BenchDesign-class]{BenchDesign}} object with 35 | the specified changes. 36 | } 37 | \examples{ 38 | ## empty BenchDesign 39 | bench <- BenchDesign() 40 | 41 | ## add method 42 | bench <- addMethod(bench, label = "qv", 43 | func = qvalue::qvalue, 44 | post = function(x) { x$qvalue }, 45 | meta = list(note = "storey's q-value"), 46 | params = rlang::quos(p = pval)) 47 | 48 | ## modify method 'meta' property of 'qv' method 49 | bench <- modifyMethod(bench, label = "qv", 50 | params = rlang::quos(bd.meta = 51 | list(note = "Storey's q-value"))) 52 | 53 | ## verify that method has been updated 54 | printMethod(bench, "qv") 55 | 56 | } 57 | \seealso{ 58 | \code{\link{addMethod}}, \code{\link{expandMethod}}, \code{\link{dropMethod}} 59 | } 60 | \author{ 61 | Patrick Kimes 62 | } 63 | -------------------------------------------------------------------------------- /man/BenchDesign.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/BenchDesign-constructor.R, 3 | % R/SummarizedBenckmark-accessors.R 4 | \docType{methods} 5 | \name{BenchDesign} 6 | \alias{BenchDesign} 7 | \alias{BenchDesign,ANY-method} 8 | \alias{BenchDesign,SummarizedBenchmark-method} 9 | \title{Create a new BenchDesign object} 10 | \usage{ 11 | BenchDesign(..., methods = NULL, data = NULL) 12 | 13 | \S4method{BenchDesign}{ANY}(..., methods = NULL, data = NULL) 14 | 15 | \S4method{BenchDesign}{SummarizedBenchmark}(methods, data) 16 | } 17 | \arguments{ 18 | \item{...}{named set of \code{BDMethod} objects and/or unnamed 19 | \code{BenchDesign} objects. Only the methods of any \code{BenchDesign} 20 | object will be used, and the data slot of the objects will be 21 | ignored.} 22 | 23 | \item{methods}{named set of \code{BDMethod} objects and/or unnamed 24 | \code{BenchDesign} objects as a list. (default = NULL)} 25 | 26 | \item{data}{optional data.frame or other list object to be 27 | used in the benchmark. (default = NULL)} 28 | } 29 | \value{ 30 | \code{BenchDesign} object. 31 | } 32 | \description{ 33 | Initializes a new BenchDesign object of benchmarking methods and data. 34 | 35 | The BenchDesign class serves as the core container 36 | for methods and data used for benchmarking in the SummarizedBenchmark package. The object 37 | can be initialized with a list of methods ot be benchmarked, a default benchmarking data set, 38 | both or neither. Methods must be passed to the constructor as \code{BDMethod} or \code{BDMethodList} 39 | objects. 40 | 41 | The constructor can also be used to access the BenchDesign stored in a SummarizedBenchmark 42 | object. 43 | } 44 | \examples{ 45 | ## with no input 46 | bd <- BenchDesign() 47 | 48 | ## with data - data must be a named argument 49 | datadf <- data.frame(pval = runif(20), x1 = rnorm(20)) 50 | bd <- BenchDesign(data = datadf) 51 | 52 | ## with two methods and data 53 | method_bh <- BDMethod(stats::p.adjust, params = rlang::quos(p = pval, method = "BH")) 54 | method_bf <- BDMethod(stats::p.adjust, params = rlang::quos(p = pval, method = "bonferroni")) 55 | bd <- BenchDesign(bh = method_bh, bonf = method_bf, 56 | data = datadf) 57 | 58 | ## with BDMethodList and data 59 | bdml <- BDMethodList(bh = method_bh, bonf = method_bf) 60 | bd <- BenchDesign(methods = bdml, data = datadf) 61 | 62 | } 63 | \seealso{ 64 | \code{\link{BenchDesign-class}}, \code{\link{BDMethod}}, \code{\link{BDMethodList}} 65 | } 66 | \author{ 67 | Patrick Kimes 68 | } 69 | -------------------------------------------------------------------------------- /man/compareBenchDesigns.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/compareBenchDesigns.R 3 | \docType{methods} 4 | \name{compareBenchDesigns} 5 | \alias{compareBenchDesigns} 6 | \alias{compareBenchDesigns,SummarizedBenchmark,missing-method} 7 | \alias{compareBenchDesigns,SummarizedBenchmark,SummarizedBenchmark-method} 8 | \alias{compareBenchDesigns,SummarizedBenchmark,BenchDesign-method} 9 | \alias{compareBenchDesigns,BenchDesign,SummarizedBenchmark-method} 10 | \alias{compareBenchDesigns,BenchDesign,BenchDesign-method} 11 | \title{Compare BenchDesign objects} 12 | \usage{ 13 | compareBenchDesigns(x, y = NULL, ...) 14 | 15 | \S4method{compareBenchDesigns}{SummarizedBenchmark,missing}(x, y = NULL, 16 | ...) 17 | 18 | \S4method{compareBenchDesigns}{SummarizedBenchmark,SummarizedBenchmark}(x, 19 | y = NULL, ...) 20 | 21 | \S4method{compareBenchDesigns}{SummarizedBenchmark,BenchDesign}(x, 22 | y = NULL, ...) 23 | 24 | \S4method{compareBenchDesigns}{BenchDesign,SummarizedBenchmark}(x, 25 | y = NULL, ...) 26 | 27 | \S4method{compareBenchDesigns}{BenchDesign,BenchDesign}(x, y = NULL, ...) 28 | } 29 | \arguments{ 30 | \item{x}{a SummarizedBenchmark or BenchDesign object} 31 | 32 | \item{y}{an optional second SummarizedBenchmark or BenchDesign object 33 | (default = NULL)} 34 | 35 | \item{...}{other parameters} 36 | } 37 | \value{ 38 | list of comparison results 39 | } 40 | \description{ 41 | Comparison of BenchDesign objects and BenchDesign method information 42 | stored in SummarizedBenchmark objects. Inputs can be either BenchDesign 43 | or SummarizedBenchmark objects. If SummarizedBenchmark objects are 44 | specified, the method metadata stored in the \code{colData} will be 45 | used for the comparison. If only a single SummarizedBenchmark object is 46 | specified, the \code{colData} information will be compared with the 47 | BenchDesign object in the \code{BenchDesign} slot of the object. 48 | To compare the \code{BenchDesign} slots of SummarizedBenchmark objects, 49 | the BenchDesigns should be extracted with \code{BenchDesign(sb)} and 50 | passed as inputs (see Examples). 51 | } 52 | \examples{ 53 | bd1 <- 54 | BenchDesign(norm_sd = BDMethod(stats::rnorm, 55 | params = rlang::quos(n = n), 56 | post = sd), 57 | t_sd = BDMethod(stats::rt, 58 | params = rlang::quos(n = n, df = 1), 59 | post = sd)) 60 | bd2 <- addMethod(bd1, "chi_sd", 61 | func = stats::rchisq, 62 | params = rlang::quos(n = n, df = 1), 63 | post = sd) 64 | 65 | compareBenchDesigns(bd1, bd2) 66 | 67 | } 68 | \seealso{ 69 | \code{\link{compareBDMethod}}, \code{\link{compareBDData}} 70 | } 71 | \author{ 72 | Patrick Kimes 73 | } 74 | -------------------------------------------------------------------------------- /R/BenchDesign-constructor.R: -------------------------------------------------------------------------------- 1 | #' Create a new BenchDesign object 2 | #' 3 | #' @description 4 | #' Initializes a new BenchDesign object of benchmarking methods and data. 5 | #' 6 | #' The BenchDesign class serves as the core container 7 | #' for methods and data used for benchmarking in the SummarizedBenchmark package. The object 8 | #' can be initialized with a list of methods ot be benchmarked, a default benchmarking data set, 9 | #' both or neither. Methods must be passed to the constructor as \code{BDMethod} or \code{BDMethodList} 10 | #' objects. 11 | #' 12 | #' The constructor can also be used to access the BenchDesign stored in a SummarizedBenchmark 13 | #' object. 14 | #' 15 | #' @param ... named set of \code{BDMethod} objects and/or unnamed 16 | #' \code{BenchDesign} objects. Only the methods of any \code{BenchDesign} 17 | #' object will be used, and the data slot of the objects will be 18 | #' ignored. 19 | #' @param methods named set of \code{BDMethod} objects and/or unnamed 20 | #' \code{BenchDesign} objects as a list. (default = NULL) 21 | #' @param data optional data.frame or other list object to be 22 | #' used in the benchmark. (default = NULL) 23 | #' 24 | #' @return 25 | #' \code{BenchDesign} object. 26 | #' 27 | #' @examples 28 | #' ## with no input 29 | #' bd <- BenchDesign() 30 | #' 31 | #' ## with data - data must be a named argument 32 | #' datadf <- data.frame(pval = runif(20), x1 = rnorm(20)) 33 | #' bd <- BenchDesign(data = datadf) 34 | #' 35 | #' ## with two methods and data 36 | #' method_bh <- BDMethod(stats::p.adjust, params = rlang::quos(p = pval, method = "BH")) 37 | #' method_bf <- BDMethod(stats::p.adjust, params = rlang::quos(p = pval, method = "bonferroni")) 38 | #' bd <- BenchDesign(bh = method_bh, bonf = method_bf, 39 | #' data = datadf) 40 | #' 41 | #' ## with BDMethodList and data 42 | #' bdml <- BDMethodList(bh = method_bh, bonf = method_bf) 43 | #' bd <- BenchDesign(methods = bdml, data = datadf) 44 | #' 45 | #' @seealso \code{\link{BenchDesign-class}}, \code{\link{BDMethod}}, \code{\link{BDMethodList}} 46 | #' @name BenchDesign 47 | #' @importFrom methods .valueClassTest 48 | #' @export 49 | #' @author Patrick Kimes 50 | NULL 51 | 52 | .BenchDesign <- function(..., methods, data) { 53 | ## add shortcut for just a single SummarizedBenchmark object (want to return w/ data) 54 | ml <- list(...) 55 | if (length(ml) == 1 && is.null(methods) && is.null(data) && is(ml[[1]], "SummarizedBenchmark")) 56 | return(BenchDesign(methods = ml[[1]])) 57 | 58 | bdml <- BDMethodList(..., x = methods) 59 | if (!is.null(data) && !is(data, "BDData")) { 60 | data <- new("BDData", data = data, 61 | type = ifelse(is(data, "character"), "md5hash", "data")) 62 | } 63 | new("BenchDesign", methods = bdml, data = data) 64 | } 65 | 66 | #' @rdname BenchDesign 67 | #' @exportMethod "BenchDesign" 68 | setMethod("BenchDesign", signature(methods = "ANY", data = "ANY"), .BenchDesign) 69 | -------------------------------------------------------------------------------- /R/DefaultMetrics.R: -------------------------------------------------------------------------------- 1 | #' @title List pre-defined metrics for SummarizedBenchmark objects 2 | #' @aliases availableMetrics 3 | #' @description 4 | #' This function returns a data frame summarizing the default performance metrics provided in this package. 5 | #' The data.frame contains three columns, `functions` is the name of the performance metric, `description` 6 | #' is longer description of the performance metric and `requiredTruth` is logical depending on whether the 7 | #' performance metrics require ground truths. 8 | #' @examples 9 | #' availableMetrics() 10 | #' 11 | #' @return A data.frame summarizing the default performance metrics provided in this package. 12 | #' @md 13 | #' @export 14 | #' @author Alejandro Reyes 15 | availableMetrics <- function(){ 16 | data.frame( 17 | functions=c( "rejections", "TPR", "TNR", "FDR", "FNR", 18 | "correlation", "sdad", "hamming", "LPnorm", 19 | "adjustedRandIndex" ), 20 | description=c("Number of rejections", "True Positive Rate", "True Negative Rate", 21 | "False Discovery Rate (estimated)", "False Negative Rate", "Pearson correlation", 22 | "Standard Deviation of the Absolute Difference", "Hamming distance", 23 | "L_{p} norm", "Adjusted Rand Index"), 24 | requiresTruth=rep( c(FALSE, TRUE), c( 1, 9 ) ) ) 25 | } 26 | 27 | sb.TPR <- function( query, truth, alpha=0.1 ){ 28 | sum( query <= alpha & truth == 1, na.rm = TRUE ) / sum( truth == 1, na.rm = TRUE ) 29 | } 30 | 31 | sb.TNR <- function( query, truth, alpha=0.1 ){ 32 | sum( ( !query <= alpha ) & truth == 0, na.rm = TRUE ) / sum( truth == 0, na.rm = TRUE ) 33 | } 34 | 35 | sb.FDR <- function( query, truth, alpha=0.1 ){ 36 | sum( query <= alpha & truth == 0, na.rm = TRUE ) / sum( query <= alpha, na.rm = TRUE ) 37 | } 38 | 39 | sb.FNR <- function( query, truth, alpha=0.1 ){ 40 | sum( !(query <= alpha) & truth == 1, na.rm = TRUE ) / sum( !( query <= alpha ), na.rm = TRUE ) 41 | } 42 | 43 | sb.rejections <- function( query, truth, alpha=0.1 ){ 44 | sum( query <= alpha, na.rm = TRUE ) 45 | } 46 | 47 | #' @importFrom S4Vectors cor 48 | sb.correlation <- function( query, truth, method="pearson" ){ 49 | cor( query, truth, method=method ) 50 | } 51 | 52 | #' @importFrom BiocGenerics sd 53 | sb.sdad <- function( query, truth ){ 54 | sd( abs( query - truth ) ) 55 | } 56 | 57 | sb.hamming <- function( query, truth ){ 58 | sum( !query == truth, na.rm = TRUE ) 59 | } 60 | 61 | sb.Lpnorm <- function( query, truth, p=2 ){ 62 | sum( abs(query - truth) ^ p, na.rm = TRUE ) ^ (1/p) 63 | } 64 | 65 | #' @importFrom mclust adjustedRandIndex 66 | sb.adjustedRandIndex <- function( query, truth ){ 67 | adjustedRandIndex( query, truth ) 68 | } 69 | 70 | assayHasTruths <- function( object, assay ){ 71 | if( is.null( rowData(object)[[assay]] ) ){ 72 | return(FALSE) 73 | } 74 | if( all( is.na( rowData( object )[[assay]] ) ) ){ 75 | return(FALSE) 76 | }else{ 77 | return(TRUE) 78 | } 79 | } 80 | 81 | -------------------------------------------------------------------------------- /vignettes/library.bib: -------------------------------------------------------------------------------- 1 | @article{Stephens_2017, 2 | author = {Stephens, Matthew}, 3 | doi = {10.1093/biostatistics/kxw041}, 4 | journal = {Biostatistics}, 5 | number = {2}, 6 | pages = {275--294}, 7 | title = {{False discovery rates: a new deal}}, 8 | volume = {18}, 9 | year = {2017} 10 | } 11 | @article{Soneson_2016, 12 | title={{iCOBRA}: open, reproducible, standardized and live method benchmarking}, 13 | volume={13}, 14 | ISSN={1548-7105}, 15 | url={http://dx.doi.org/10.1038/nmeth.3805}, 16 | DOI={10.1038/nmeth.3805}, 17 | number={4}, 18 | journal={Nature Methods}, 19 | author={Soneson, Charlotte and Robinson, Mark D}, 20 | year={2016}, 21 | month={Mar}, 22 | pages={283–283} 23 | } 24 | @article{Li_2017, 25 | title={A Comprehensive Mouse Transcriptomic BodyMap across 17 Tissues by RNA-seq}, 26 | volume={7}, 27 | ISSN={2045-2322}, 28 | url={http://dx.doi.org/10.1038/s41598-017-04520-z}, 29 | DOI={10.1038/s41598-017-04520-z}, 30 | number={1}, 31 | journal={Scientific Reports}, 32 | author={Li, Bin and Qing, Tao and Zhu, Jinhang and Wen, Zhuo and Yu, Ying and Fukumura, Ryutaro and Zheng, Yuanting and Gondo, Yoichi and Shi, Leming}, 33 | year={2017} 34 | } 35 | @article{Teng_2016, 36 | title={A benchmark for RNA-seq quantification pipelines}, 37 | volume={17}, 38 | ISSN={1474-760X}, 39 | url={http://dx.doi.org/10.1186/s13059-016-0940-1}, 40 | DOI={10.1186/s13059-016-0940-1}, 41 | number={1}, 42 | journal={Genome Biology}, 43 | author={Teng, Mingxiang and Love, Michael I. and Davis, Carrie A. and Djebali, Sarah and Dobin, Alexander and Graveley, Brenton R. and Li, Sheng and Mason, Christopher E. and Olson, Sara and Pervouchine, Dmitri and et al.}, 44 | year={2016} 45 | } 46 | @article{Zappia_2017, 47 | author = {Luke Zappia and Belinda Phipson and Alicia Oshlack}, 48 | title = {Splatter: simulation of single-cell RNA sequencing data}, 49 | journal = {Genome Biology}, 50 | year = {2017}, 51 | url = {http://dx.doi.org/10.1186/s13059-017-1305-0}, 52 | doi = {10.1186/s13059-017-1305-0} 53 | } 54 | @article{Dunn_1961, 55 | title = {Multiple Comparisons Among Means}, 56 | author = {Dunn, Olive Jean}, 57 | journal = {Journal of the American Statistical Association}, 58 | volume = {56}, 59 | number = {293}, 60 | pages = {52--64}, 61 | year = {1961} 62 | } 63 | @article{Benjamini_1995, 64 | title={Controlling the false discovery rate: a practical and powerful approach to multiple testing}, 65 | author = {Benjamini, Yoav and Hochberg, Yosef}, 66 | journal = {Journal of the Royal Statistical Society. Series B (Methodological)}, 67 | pages = {289--300}, 68 | year = {1995} 69 | } 70 | @article{Storey_2002, 71 | title = {A direct approach to false discovery rates}, 72 | author = {Storey, John D}, 73 | journal = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)}, 74 | volume = {64}, 75 | number = {3}, 76 | pages = {479--498}, 77 | year = {2002} 78 | } 79 | -------------------------------------------------------------------------------- /tests/testthat/test_estimatePerformanceMetrics.R: -------------------------------------------------------------------------------- 1 | context("estimatePeformanceMetrics") 2 | test_that("test='estimatePerformanceMetrics' returns informative messages", { 3 | data( sb ) 4 | expect_error( estimatePerformanceMetrics( 1:10 ) ) 5 | expect_error( estimatePerformanceMetrics( sb ) ) 6 | sb <- addPerformanceMetric( sb, evalMetric=c("rejections", "TPR", "TNR", "FDR", "FNR"), assay="qvalue" ) 7 | expect_s4_class( estimatePerformanceMetrics( sb ), "DataFrame" ) 8 | tidyRes1 <- estimatePerformanceMetrics( sb, tidy=TRUE ) 9 | expect_s3_class( tidyRes1, "data.frame" ) 10 | sb <- estimatePerformanceMetrics( sb, addColData=TRUE ) 11 | tidyRes2 <- tidyUpMetrics( sb ) 12 | expect_identical( tidyRes1, tidyRes2 ) 13 | expect_message( estimatePerformanceMetrics(sb) ) 14 | } ) 15 | 16 | test_that("test='estimatePerformanceMetrics' works with non scalar outputs", { 17 | data( sb ) 18 | sb2 <- sb 19 | sb2 <- addPerformanceMetric( 20 | object=sb2, 21 | assay="qvalue", 22 | evalMetric="TPR", 23 | evalFunction = function( query, truth, alpha=0.1 ){ 24 | goodHits <- sum( (query < alpha) & truth == 1 ) 25 | goodHits / sum(truth == 1) 26 | c(goodHits, goodHits) 27 | } 28 | ) 29 | expect_s4_class( estimatePerformanceMetrics(sb2, alpha=0.1 ), "DataFrame" ) 30 | 31 | data( sb ) 32 | sb2 <- sb 33 | sb2 <- addPerformanceMetric( 34 | object=sb2, 35 | assay="qvalue", 36 | evalMetric="TPR", 37 | evalFunction = function( query, truth, alpha=0.1 ){ 38 | goodHits <- sum( (query < alpha) & truth == 1 ) 39 | goodHits / sum(truth == 1) 40 | c(goodHits, goodHits) 41 | } 42 | ) 43 | expect_s4_class( estimatePerformanceMetrics(sb2, alpha=c(0.1, 0.2) ), "DataFrame" ) 44 | 45 | data( sb ) 46 | sb2 <- sb 47 | sb2 <- addPerformanceMetric( 48 | object=sb2, 49 | assay="qvalue", 50 | evalMetric="TPR", 51 | evalFunction = function( query, truth ){ 52 | goodHits <- sum(truth == 1) 53 | c(goodHits, goodHits, goodHits) 54 | } 55 | ) 56 | expect_s4_class( estimatePerformanceMetrics(sb2, alpha=c(0.1, 0.2)), "DataFrame" ) 57 | 58 | data( sb ) 59 | sb2 <- sb 60 | sb2 <- addPerformanceMetric( 61 | object=sb2, 62 | assay="qvalue", 63 | evalMetric="TPR", 64 | evalFunction = function( query, truth, alpha=0.1 ){ 65 | goodHits <- sum( (query < alpha) & truth == 1 ) 66 | goodHits / sum(truth == 1) 67 | } 68 | ) 69 | expect_s4_class( estimatePerformanceMetrics(sb2, alpha=c(0.1, 0.2)), "DataFrame" ) 70 | 71 | data( sb ) 72 | sb2 <- sb 73 | sb2 <- addPerformanceMetric( 74 | object=sb2, 75 | assay="qvalue", 76 | evalMetric="TPR", 77 | evalFunction = function( query, truth, alpha=0.1 ){ 78 | goodHits <- sum( (query < alpha) & truth == 1 ) 79 | goodHits / sum(truth == 1) 80 | c(SummarizedExperiment(), SummarizedExperiment()) 81 | } 82 | ) 83 | expect_s4_class( estimatePerformanceMetrics(sb2, alpha=c(0.1, 0.2)), "DataFrame" ) 84 | } ) 85 | -------------------------------------------------------------------------------- /man/SummarizedBenchmark.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/SummarizedBenckmark-constructor.R 3 | \name{SummarizedBenchmark} 4 | \alias{SummarizedBenchmark} 5 | \title{Create a new SummarizedBenchmark object} 6 | \usage{ 7 | SummarizedBenchmark(assays, colData, ftData = NULL, groundTruth = NULL, 8 | performanceMetrics = NULL, BenchDesign = NULL, ...) 9 | } 10 | \arguments{ 11 | \item{assays}{A list containing outputs of the methods to be 12 | benchmark. Each element of the list must contain a matrix or 13 | data.frame of n x m, n being the number of features tested 14 | (e.g. genes) and m being the number of methods in the benchmark. 15 | Each element of the list must contain a single assay 16 | (the outputs of the methods). For example, 17 | for a benchmark of differential expression methods, one assay could contain 18 | the q-values from the different methods and another assay could be the 19 | estimated log fold changes.} 20 | 21 | \item{colData}{A \code{\link{DataFrame}} describing the annotation of the 22 | methods. These could include version of the software or the parameters 23 | used to run them.} 24 | 25 | \item{ftData}{A \code{\link{DataFrame}} object describing the rows. This parameter 26 | is equivalent to the parameter rowData of a SummarizedExperiment.} 27 | 28 | \item{groundTruth}{If present, a \code{\link{DataFrame}} containing the ground truths. 29 | If provided, the number of columns must be the same as the number of assays 30 | (NA's are accepted). 31 | The names of the columns should have the same names as the assays.} 32 | 33 | \item{performanceMetrics}{A \code{\link{SimpleList}} of the same 34 | length as the number of assays. Each element of the list must be a list 35 | of functions. Each function must contain the parameters 'query' and 36 | 'truth'.} 37 | 38 | \item{BenchDesign}{A \code{\link{BenchDesign}} containing the code used 39 | to construct the object. (default = NULL)} 40 | 41 | \item{...}{Additional parameters passed to \code{\link{SummarizedExperiment}}.} 42 | } 43 | \value{ 44 | A \code{\link{SummarizedBenchmark}} object. 45 | } 46 | \description{ 47 | Function to construct \code{SummarizedBenchmark} objects. 48 | } 49 | \examples{ 50 | 51 | ## loading the example data from iCOBRA 52 | library(iCOBRA) 53 | data(cobradata_example) 54 | 55 | ## a bit of data wrangling and reformatting 56 | assays <- list( 57 | qvalue=cobradata_example@padj, 58 | logFC=cobradata_example@score ) 59 | assays[["qvalue"]]$DESeq2 <- p.adjust(cobradata_example@pval$DESeq2, method="BH") 60 | groundTruth <- DataFrame( cobradata_example@truth[,c("status", "logFC")] ) 61 | colnames(groundTruth) <- names( assays ) 62 | colData <- DataFrame( method=colnames(assays[[1]]) ) 63 | groundTruth <- groundTruth[rownames(assays[[1]]),] 64 | 65 | ## constructing a SummarizedBenchmark object 66 | sb <- SummarizedBenchmark( 67 | assays=assays, colData=colData, 68 | groundTruth=groundTruth ) 69 | colData(sb)$label <- rownames(colData(sb)) 70 | 71 | } 72 | \author{ 73 | Alejandro Reyes 74 | } 75 | -------------------------------------------------------------------------------- /man/expandMethod.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/expandMethod.R 3 | \name{expandMethod} 4 | \alias{expandMethod} 5 | \title{Expand method in BenchDesign object} 6 | \usage{ 7 | expandMethod(bd, label, params, onlyone = NULL, .replace = FALSE, 8 | .overwrite = FALSE) 9 | } 10 | \arguments{ 11 | \item{bd}{\code{\link[=BenchDesign-class]{BenchDesign}} object.} 12 | 13 | \item{label}{Character name of method to be expanded.} 14 | 15 | \item{params}{Named list of quosure lists specifying the label of the 16 | new methods to be added to the \code{\link[=BenchDesign-class]{BenchDesign}}, 17 | and the set of 18 | parameters to overwrite in the original method definition for 19 | each new method. Alternatively, if \code{onlyone} is non-\code{NULL}, a single quosure 20 | list with \code{name = value} pairs specifying the label of the new methods and 21 | the values to use for overwriting the parameter specified in \code{onlyone}.} 22 | 23 | \item{onlyone}{Character name of a parameter to be modified. Only specify 24 | if just a single parameter should be replaced in the original 25 | method definition. Ignored if \code{NULL}. (default = \code{NULL})} 26 | 27 | \item{.replace}{Logical whether original \code{label} method should be removed. 28 | (default = \code{FALSE})} 29 | 30 | \item{.overwrite}{Logical whether to overwrite the existing list of 31 | parameters (\code{TRUE}) or to simply add the new parameters to the existing 32 | list (\code{FALSE}). (default = \code{FALSE})} 33 | } 34 | \value{ 35 | Modified \code{\link[=BenchDesign-class]{BenchDesign}} object with new methods with 36 | specified parameters added. 37 | } 38 | \description{ 39 | Takes a \code{\link[=BenchDesign-class]{BenchDesign}} object, the name of an 40 | existing method, and new parameter specifications, 41 | and returns a modified \code{\link[=BenchDesign-class]{BenchDesign}} 42 | object with new methods added. The named method is "expanded" to mutliple methods 43 | according to the specified set of parameters. 44 | } 45 | \examples{ 46 | ## empty BenchDesign 47 | bench <- BenchDesign() 48 | 49 | ## add basic 'padjust' method 50 | bench <- addMethod(bench, label = "padjust", 51 | func = p.adjust, 52 | params = rlang::quos(p = pval, method = "none")) 53 | 54 | ## modify multiple parameters - params is a list of quosure lists 55 | newparams <- list(bonf = rlang::quos(p = round(pval, 5), method = "bonferonni"), 56 | bh = rlang::quos(p = round(pval, 3), method = "BH")) 57 | bench_exp <- expandMethod(bench, label = "padjust", params = newparams) 58 | BDMethodList(bench_exp) 59 | 60 | ## only modify a single parameter - params is a quosure list 61 | newparams <- rlang::quos(bonf = "bonferonni", BH = "BH") 62 | bench_exp <- expandMethod(bench, label = "padjust", onlyone = "method", params = newparams) 63 | BDMethodList(bench_exp) 64 | 65 | } 66 | \seealso{ 67 | \code{\link{modifyMethod}}, \code{\link{addMethod}}, \code{\link{dropMethod}} 68 | } 69 | \author{ 70 | Patrick Kimes 71 | } 72 | -------------------------------------------------------------------------------- /man/BDMethod.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/AllGenerics.R, R/BDMethod-constructor.R, 3 | % R/BDMethodList-accessors.R, R/BenchDesign-accessors.R 4 | \docType{methods} 5 | \name{BDMethod} 6 | \alias{BDMethod} 7 | \alias{BDMethod,quosure-method} 8 | \alias{BDMethod,function-method} 9 | \alias{BDMethod,BDMethodList-method} 10 | \alias{BDMethod,BenchDesign-method} 11 | \title{Create a new BDMethod object} 12 | \usage{ 13 | BDMethod(x, params = rlang::quos(), post = NULL, meta = NULL, ...) 14 | 15 | \S4method{BDMethod}{quosure}(x, params = rlang::quos(), post = NULL, 16 | meta = NULL, ...) 17 | 18 | \S4method{BDMethod}{`function`}(x, params = rlang::quos(), post = NULL, 19 | meta = NULL, ...) 20 | 21 | \S4method{BDMethod}{BDMethodList}(x, i = 1) 22 | 23 | \S4method{BDMethod}{BenchDesign}(x, i = 1) 24 | } 25 | \arguments{ 26 | \item{x}{main method function or function quosure. Alternative, may be a 27 | \code{\link[=BDMethodList-class]{BDMethodList}} or 28 | \code{\link[=BenchDesign-class]{BenchDesign}} object from which 29 | the \code{\link[=BDMethod-class]{BDMethod}} should be 30 | extracted.} 31 | 32 | \item{params}{list of quosures specifying function parameters. 33 | (default = \code{rlang::quos()})} 34 | 35 | \item{post}{list of functions to be applied to the output of \code{x}. 36 | (default = NULL)} 37 | 38 | \item{meta}{list of metadata. (default = NULL)} 39 | 40 | \item{...}{other parameters.} 41 | 42 | \item{i}{integer index or character name of \code{\link[=BDMethod-class]{BDMethod}} in 43 | \code{\link[=BDMethodList-class]{BDMethodList}} or 44 | \code{\link[=BenchDesign-class]{BenchDesign}} object.} 45 | } 46 | \value{ 47 | BDMethod object 48 | } 49 | \description{ 50 | Initializes a new BenchDesign method object for benchmarking. 51 | 52 | New \code{\link[=BDMethod-class]{BDMethod}} objects are typically not directly constructed as they 53 | have limited use outside of \code{\link[=BenchDesign-class]{BenchDesign}} objects. Instead, 54 | methods in a \code{\link[=BenchDesign-class]{BenchDesign}} object are more commonly created, modified or 55 | removed using 56 | function calls on the \code{\link[=BenchDesign-class]{BenchDesign}}, e.g. using \code{\link{addMethod}} to 57 | add a new method object. 58 | 59 | The constructor can also be used to access \code{\link[=BDMethod-class]{BDMethod}} objects stored in 60 | \code{\link[=BDMethodList-class]{BDMethodList}} and \code{\link[=BenchDesign-class]{BenchDesign}} objects. 61 | } 62 | \examples{ 63 | ## create a simple BDMethod 64 | bdm1 <- BDMethod(x = base::mean) 65 | 66 | ## create a more complex BDMethod 67 | bdm2 <- BDMethod(x = function(x) { x^2 }, post = base::sqrt, 68 | meta = list(note = "simple example")) 69 | 70 | ## construct a BenchDesign with the BDMethod objects 71 | bd <- BenchDesign(method1 = bdm1, method2 = bdm2) 72 | 73 | ## access a BDMethod in the BenchDesign 74 | BDMethod(bd, "method1") 75 | 76 | } 77 | \seealso{ 78 | \code{\link{BDMethod-class}}, \code{\link{BenchDesign}}, \code{\link{BDMethodList}} 79 | } 80 | \author{ 81 | Patrick Kimes 82 | } 83 | -------------------------------------------------------------------------------- /man/updateBench.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/updateBench.R 3 | \name{updateBench} 4 | \alias{updateBench} 5 | \title{Check/Update SummarizedBenchmark} 6 | \usage{ 7 | updateBench(sb, bd = NULL, dryrun = TRUE, version = FALSE, 8 | keepAll = TRUE, reuseParams = TRUE, ...) 9 | } 10 | \arguments{ 11 | \item{sb}{a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object} 12 | 13 | \item{bd}{a \code{\link[=BenchDesign-class]{BenchDesign}} object} 14 | 15 | \item{dryrun}{logical whether to just print description of what would 16 | be updated rather than actually running any methods. (default = TRUE)} 17 | 18 | \item{version}{logical whether to re-run methods with only package 19 | version differences. (default = FALSE)} 20 | 21 | \item{keepAll}{logical whether to keep methods run in original \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 22 | but not in new \code{\link[=BenchDesign-class]{BenchDesign}}. Only used if \code{bd} is not NULL. (default = TRUE)} 23 | 24 | \item{reuseParams}{logical whether to reuse parameters from \code{buildBench} call 25 | used to create \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object (if available). Directly 26 | specified \code{\link{buildBench}} parameters still take precedence. (default = TRUE)} 27 | 28 | \item{...}{optional parameters to pass to \code{\link{buildBench}}.} 29 | } 30 | \value{ 31 | SumamrizedBenchmark object. 32 | } 33 | \description{ 34 | Function to update or check status of \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 35 | results. 36 | 37 | If only a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object is specified, 38 | the function will check whether `func`, `param`, `meta`, `post` or the `pkg_vers` of the 39 | methods in the \code{\link[=BenchDesign-class]{BenchDesign}} stored with the 40 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} do not match values stored in the 41 | \code{colData}. By default, no methods will be executed to update results. To actually execute updates, 42 | set \code{dryrun = FALSE}. 43 | 44 | If a \code{\link[=BenchDesign-class]{BenchDesign}} object is specified in addition to a 45 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object, the function will check which 46 | methods in the new \code{\link[=BenchDesign-class]{BenchDesign}} need to be executed to update the 47 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} results. Again, by default, no methods 48 | will be executed unless \code{dryrun = FLASE} is specified. 49 | 50 | Unless \code{reuseParams = FALSE} is specified, the parameters of the last execution session stored 51 | in the the \code{colData} of the \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object 52 | will be used. 53 | } 54 | \examples{ 55 | ## load example SummarizedBenchmark object 56 | data(allSB) 57 | sb <- allSB[[1]] 58 | 59 | ## check if results are out of date 60 | updateBench(sb) 61 | 62 | ## modify BenchDesign 63 | bd <- BenchDesign(sb) 64 | bd <- dropMethod(bd, "kallisto-default") 65 | 66 | ## check if results need to be updated with new BenchDesign 67 | updateBench(sb, bd) 68 | 69 | } 70 | \seealso{ 71 | \code{\link{buildBench}} 72 | } 73 | \author{ 74 | Patrick Kimes 75 | } 76 | -------------------------------------------------------------------------------- /man/estimateMetrics.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/PerformanceMetrics.R 3 | \name{estimateMetricsForAssay} 4 | \alias{estimateMetricsForAssay} 5 | \alias{estimatePerformanceMetrics} 6 | \title{Estimate performance metrics in SummarizedBenchmark object} 7 | \usage{ 8 | estimateMetricsForAssay(object, assay, evalMetric = NULL, 9 | addColData = FALSE, evalFunction = NULL, tidy = FALSE, ...) 10 | 11 | estimatePerformanceMetrics(object, addColData = FALSE, tidy = FALSE, 12 | rerun = TRUE, ...) 13 | } 14 | \arguments{ 15 | \item{object}{A \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object.} 16 | 17 | \item{assay}{A string with an assay name. Indicates the assay that should be 18 | given as input to this performance metric.} 19 | 20 | \item{evalMetric}{A string with the name of the evaluation metric.} 21 | 22 | \item{addColData}{Logical (default: FALSE). If TRUE, the results are added to the 23 | \code{\link{colData}} slot of the \code{\link{SummarizedExperiment}} object and 24 | the object is returned. If FALSE, only a \code{\link{DataFrame}} with the results 25 | is returned.} 26 | 27 | \item{evalFunction}{A function that calculates a performance metric. It should 28 | contain at least two arguments, query and truth, where query is the output vector 29 | of a method and truth is the vector of ground true values. If additional parameters 30 | are specified, they must contain default values. If this parameter is passed, 31 | the metrics in the object are ignored and only this evaluation metric 32 | is estimated.} 33 | 34 | \item{tidy}{Logical (default: FALSE). If TRUE, a long formated \code{\link{data.frame}} 35 | is returned.} 36 | 37 | \item{...}{Additional parameters passed to the performance functions.} 38 | 39 | \item{rerun}{Logical (default: TRUE). By default, all performance metrics are recalculated 40 | everytime that \code{\link{estimatePerformanceMetrics}} is called. If FALSE, performance metrics 41 | will only be calculated for newly added methods or modified methods.} 42 | } 43 | \value{ 44 | Either a \code{\link{SummarizedBenchmark}} object, a \code{\link{DataFrame}} or 45 | a \code{\link{data.frame}}. 46 | } 47 | \description{ 48 | These functions estimate the performance metrics, either passed as arguments or 49 | added previously with the \code{\link{addPerformanceMetric}} function. The function 50 | will estimate the performance metric for each method. 51 | } 52 | \section{Functions}{ 53 | \itemize{ 54 | \item \code{estimateMetricsForAssay}: Estimate performance metrics for a given assay 55 | 56 | \item \code{estimatePerformanceMetrics}: Estimate performance metrics for all assays 57 | }} 58 | 59 | \examples{ 60 | 61 | data( sb ) 62 | sb <- addPerformanceMetric( 63 | object=sb, 64 | assay="qvalue", 65 | evalMetric="TPR", 66 | evalFunction = function( query, truth, alpha=0.1 ){ 67 | goodHits <- sum( (query < alpha) & truth == 1 ) 68 | goodHits / sum(truth == 1) 69 | } 70 | ) 71 | 72 | qvalueMetrics <- estimateMetricsForAssay( sb, assay="qvalue" ) 73 | allMetrics <- estimatePerformanceMetrics( sb ) 74 | allMetricsTidy <- estimatePerformanceMetrics( sb, tidy=TRUE ) 75 | 76 | } 77 | \seealso{ 78 | \code{\link{availableMetrics}}, \code{\link{performanceMetrics}} 79 | } 80 | \author{ 81 | Alejandro Reyes 82 | } 83 | -------------------------------------------------------------------------------- /R/PlottingFunctions.R: -------------------------------------------------------------------------------- 1 | #' @title Plot UpSetR for SummarizedBenchmark object 2 | #' @aliases plotMethodsOverlap 3 | #' @description 4 | #' This function looks for an assay, called by default 'qvalue', and given an alpha threshold, 5 | #' it binarizes the assay matrix depending on whether its values are below the alpha 6 | #' threshold. Then it uses the function \code{\link{upset}} to plot the overlaps. 7 | #' The plot is only generated if at least 2 methods have observations that pass 8 | #' the alpha threshold. 9 | #' 10 | #' @param object A \code{\link{SummarizedBenchmark}} object. 11 | #' @param assay The name of an assay. 12 | #' @param alpha An alpha value. 13 | #' @param ... Further arguments passed to \code{\link{upset}} 14 | #' 15 | #' @examples 16 | #' data(sb) 17 | #' \dontrun{ 18 | #' plotMethodsOverlap(sb) 19 | #' } 20 | #' 21 | #' @return An upseR plot. 22 | #' 23 | #' @seealso \code{\link{plotROC}}, \code{\link{estimatePerformanceMetrics}} 24 | #' @author Alejandro Reyes 25 | #' @export 26 | #' @importFrom UpSetR upset 27 | #' 28 | plotMethodsOverlap <- function( object, assay="qvalue", alpha=0.1, ... ){ 29 | stopifnot( is( object, "SummarizedExperiment" ) ) 30 | stopifnot( is( alpha, "numeric") ) 31 | stopifnot( alpha >=0 & alpha <= 1 ) 32 | if( !( assay %in% assayNames( object ) ) ){ 33 | stop(sprintf("Assay name '%s' not found", assay) ) 34 | } 35 | uobj <- as.data.frame( 1*( assays( object )[[assay]] < alpha) ) 36 | uobj[is.na( uobj )] <- 0 37 | if ( sum(colSums(uobj) > 0) < 2 ){ 38 | stop("To plot overlaps, at least 2 methods must have observations that pass the alpha threshold.") 39 | } 40 | upset(uobj , ... ) 41 | } 42 | 43 | #' @title Plot ROC curve for SummarizedBenchmark object 44 | #' @aliases plotROC 45 | #' @description 46 | #' This function inputs a \code{\link{SummarizedBenchmark}} object, looks 47 | #' for an assay called 'qvalue' and plots receiver operating characteristic curves 48 | #' for each of the methods to benchmark. 49 | #' 50 | #' @param object A \code{\link{SummarizedBenchmark}} object. 51 | #' @param assay An assay name. 52 | #' 53 | #' @examples 54 | #' data(sb) 55 | #' \dontrun{ 56 | #' plotROC(sb) 57 | #' } 58 | #' 59 | #' @return A ggplot object. 60 | #' 61 | #' @seealso \code{\link{plotMethodsOverlap}}, \code{\link{estimatePerformanceMetrics}} 62 | #' @author Alejandro Reyes 63 | #' @export 64 | #' @importFrom ggplot2 ggplot geom_line aes geom_abline xlim ylim 65 | #' 66 | plotROC <- function( object, assay="qvalue" ){ 67 | stopifnot( is( object, "SummarizedBenchmark" )) 68 | if( !any( assayNames( object ) %in% assay ) ){ 69 | stop(sprintf("Assay '%s' not found.\n", assay) ) 70 | } 71 | if( !assayHasTruths( object, assay ) ){ 72 | stop(sprintf("Ground truths not found for assay '%s'", assay)) 73 | } 74 | metricsDf <- do.call( rbind, lapply( colnames(object), function(method){ 75 | or <- order( assays(object)[[assay]][,method] ) 76 | TPR <- cumsum( groundTruths( object )[[assay]][or] ) / sum( groundTruths(object)[[assay]] ) 77 | FDR <- cumsum( abs(groundTruths( object )[[assay]][or] - 1) ) / seq_along(or) 78 | data.frame( method=method, TPR=TPR, FDR=FDR ) 79 | } ) ) 80 | pl <- ggplot( metricsDf, aes(FDR, TPR, col=method) ) + 81 | geom_line(size=1.2, alpha=.6) + xlim(0, 1) + ylim(0, 1) + geom_abline(linetype="dashed") 82 | pl 83 | } 84 | -------------------------------------------------------------------------------- /R/BDMethod-constructor.R: -------------------------------------------------------------------------------- 1 | #' Create a new BDMethod object 2 | #' 3 | #' @description 4 | #' Initializes a new BenchDesign method object for benchmarking. 5 | #' 6 | #' New \code{\link[=BDMethod-class]{BDMethod}} objects are typically not directly constructed as they 7 | #' have limited use outside of \code{\link[=BenchDesign-class]{BenchDesign}} objects. Instead, 8 | #' methods in a \code{\link[=BenchDesign-class]{BenchDesign}} object are more commonly created, modified or 9 | #' removed using 10 | #' function calls on the \code{\link[=BenchDesign-class]{BenchDesign}}, e.g. using \code{\link{addMethod}} to 11 | #' add a new method object. 12 | #' 13 | #' The constructor can also be used to access \code{\link[=BDMethod-class]{BDMethod}} objects stored in 14 | #' \code{\link[=BDMethodList-class]{BDMethodList}} and \code{\link[=BenchDesign-class]{BenchDesign}} objects. 15 | #' 16 | #' @param x main method function or function quosure. Alternative, may be a 17 | #' \code{\link[=BDMethodList-class]{BDMethodList}} or 18 | #' \code{\link[=BenchDesign-class]{BenchDesign}} object from which 19 | #' the \code{\link[=BDMethod-class]{BDMethod}} should be 20 | #' extracted. 21 | #' @param params list of quosures specifying function parameters. 22 | #' (default = \code{rlang::quos()}) 23 | #' @param post list of functions to be applied to the output of \code{x}. 24 | #' (default = NULL) 25 | #' @param meta list of metadata. (default = NULL) 26 | #' @param i integer index or character name of \code{\link[=BDMethod-class]{BDMethod}} in 27 | #' \code{\link[=BDMethodList-class]{BDMethodList}} or 28 | #' \code{\link[=BenchDesign-class]{BenchDesign}} object. 29 | #' @param ... other parameters. 30 | #' 31 | #' @return 32 | #' BDMethod object 33 | #' 34 | #' @examples 35 | #' ## create a simple BDMethod 36 | #' bdm1 <- BDMethod(x = base::mean) 37 | #' 38 | #' ## create a more complex BDMethod 39 | #' bdm2 <- BDMethod(x = function(x) { x^2 }, post = base::sqrt, 40 | #' meta = list(note = "simple example")) 41 | #' 42 | #' ## construct a BenchDesign with the BDMethod objects 43 | #' bd <- BenchDesign(method1 = bdm1, method2 = bdm2) 44 | #' 45 | #' ## access a BDMethod in the BenchDesign 46 | #' BDMethod(bd, "method1") 47 | #' 48 | #' @seealso \code{\link{BDMethod-class}}, \code{\link{BenchDesign}}, \code{\link{BDMethodList}} 49 | #' @name BDMethod 50 | #' @importFrom rlang get_expr eval_tidy quos !! !!! 51 | #' @importFrom methods .valueClassTest 52 | #' @export 53 | #' @author Patrick Kimes 54 | NULL 55 | 56 | .BDMethod.quo <- function(x, params, post, meta, ...) { 57 | fc <- rlang::get_expr(x) 58 | f <- rlang::eval_tidy(x) 59 | if (is(post, "function")) { 60 | post <- list("default" = post) 61 | } 62 | new("BDMethod", f = f, fc = fc, params = params, post = post, meta = meta) 63 | } 64 | 65 | .BDMethod.fun <- function(x, params, post, meta, ...) { 66 | fc <- substitute(x) 67 | if (is(post, "function")) { 68 | post <- list("default" = post) 69 | } 70 | new("BDMethod", f = x, fc = fc, params = params, post = post, meta = meta) 71 | } 72 | 73 | 74 | #' @rdname BDMethod 75 | #' @export 76 | setMethod("BDMethod", signature(x = "quosure"), .BDMethod.quo) 77 | 78 | #' @rdname BDMethod 79 | #' @export 80 | setMethod("BDMethod", signature(x = "function"), .BDMethod.fun) 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # SummarizedBenchmark 3 | 4 | [![Bioc Devel build 5 | results](https://bioconductor.org/shields/build/devel/bioc/SummarizedBenchmark.svg)](http://bioconductor.org/checkResults/devel/bioc-LATEST/SummarizedBenchmark/) 6 | [![Whether the package is available on all 7 | platforms.](https://bioconductor.org/shields/availability/3.8/SummarizedBenchmark.svg)](http://bioconductor.org/packages/devel/SummarizedBenchmark/) 8 | [![How long since the package was first in a released Bioconductor 9 | version (or is it in devel 10 | only).](https://bioconductor.org/shields/years-in-bioc/SummarizedBenchmark.svg)](http://bioconductor.org/packages/devel/SummarizedBenchmark/) 11 | [![time since last commit. possible values: today, \< 1 week, \< 1 12 | month, \< 3 months, since release, before 13 | release](https://bioconductor.org/shields/lastcommit/devel/bioc/SummarizedBenchmark.svg)](http://bioconductor.org/checkResults/devel/bioc-LATEST/SummarizedBenchmark/) 14 | 15 | SummarizedBenchmark defines a flexible framework for benchmarking 16 | computational methods in R. Classes and functions are provided for 17 | defining, executing and evaluating benchmark experiments. The package 18 | builds on the 19 | [SummarizedExperiment](http://bioconductor.org/packages/SummarizedExperiment/) 20 | class to keep results organized, with outputs tied directly with 21 | important method metadata. This site is for the development version of 22 | the package. Documentation and examples for the current Bioconductor 23 | release version of the package can be found at the official [release 24 | page](http://bioconductor.org/packages/SummarizedBenchmark/). 25 | 26 | If you have any suggestions on how we can improve the package, [let us 27 | know](https://github.com/areyesq89/SummarizedBenchmark/issues)\! 28 | 29 | ## Installation 30 | 31 | ``` r 32 | # Install development version from Bioconductor 33 | BiocManager::install("SummarizedBenchmark", version = "devel") 34 | 35 | # Install release version from Bioconductor 36 | BiocManager::install("SummarizedBenchmark") 37 | ``` 38 | 39 | ## Usage 40 | 41 | SummarizedBenchmark can be used to apply several computational methods 42 | in R on a data set and to store and compare the subsequent results. This 43 | can include just a single method with different parameter settings, or 44 | methods written outside of R, but called using a system call from an R 45 | session. Generally, the package should be useful if you are trying to 46 | decide between several competing methods and would like to compare how 47 | they perform on one or more data sets. 48 | 49 | ## Related Work 50 | 51 | While we hope users find SummarizedBenchmark useful, it might not be 52 | suitable for all problems or data sets. Other frameworks for 53 | benchmarking methods include 54 | [iCOBRA](http://bioconductor.org/packages/iCOBRA/) (a package for 55 | comparing results of “binary classification and ranking methods” with a 56 | Shiny web application for interactive analyses), 57 | [rnaseqcomp](http://bioconductor.org/packages/rnaseqcomp) (a package for 58 | comparing results of RNA-seq quantification pipelines), and 59 | [dsc](https://github.com/stephenslab/dsc) (a framework for “managing 60 | computational benchmarking experiments that compare several competing 61 | methods” written in Python but capable of running methods implemented in 62 | both Python and R). 63 | -------------------------------------------------------------------------------- /README.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | output: github_document 3 | --- 4 | 5 | # SummarizedBenchmark 6 | 7 | [![Bioc Devel build results](https://bioconductor.org/shields/build/devel/bioc/SummarizedBenchmark.svg)](http://bioconductor.org/checkResults/devel/bioc-LATEST/SummarizedBenchmark/) 8 | [![Whether the package is available on all platforms.](https://bioconductor.org/shields/availability/3.8/SummarizedBenchmark.svg)](http://bioconductor.org/packages/devel/SummarizedBenchmark/) 9 | [![How long since the package was first in a released Bioconductor version (or is it in devel only).](https://bioconductor.org/shields/years-in-bioc/SummarizedBenchmark.svg)](http://bioconductor.org/packages/devel/SummarizedBenchmark/) 10 | [![time since last commit. possible values: today, < 1 week, < 1 month, < 3 months, since release, before release](https://bioconductor.org/shields/lastcommit/devel/bioc/SummarizedBenchmark.svg)](http://bioconductor.org/checkResults/devel/bioc-LATEST/SummarizedBenchmark/) 11 | 12 | SummarizedBenchmark defines a flexible framework for benchmarking computational methods in R. 13 | Classes and functions are provided for defining, executing and evaluating 14 | benchmark experiments. The package builds on the [SummarizedExperiment](http://bioconductor.org/packages/SummarizedExperiment/) 15 | class to keep results organized, with outputs tied directly with important method metadata. 16 | This site is for the development version of the package. 17 | Documentation and examples for the current Bioconductor release version of the package 18 | can be found at the official [release page](http://bioconductor.org/packages/SummarizedBenchmark/). 19 | 20 | If you have any suggestions on how we can improve the package, 21 | [let us know](https://github.com/areyesq89/SummarizedBenchmark/issues)! 22 | 23 | ## Installation 24 | 25 | ```{r installation, eval = FALSE} 26 | # Install development version from Bioconductor 27 | BiocManager::install("SummarizedBenchmark", version = "devel") 28 | 29 | # Install release version from Bioconductor 30 | BiocManager::install("SummarizedBenchmark") 31 | ``` 32 | 33 | ## Usage 34 | 35 | SummarizedBenchmark can be used to apply several computational methods 36 | in R on a data set and to store and compare the subsequent results. This 37 | can include just a single method with different parameter settings, or 38 | methods written outside of R, but called using a system call from an 39 | R session. Generally, the package should be useful if you are trying to 40 | decide between several competing methods and would like to compare how 41 | they perform on one or more data sets. 42 | 43 | ## Related Work 44 | 45 | While we hope users find SummarizedBenchmark useful, it might not be 46 | suitable for all problems or data sets. Other frameworks for benchmarking 47 | methods include [iCOBRA](http://bioconductor.org/packages/iCOBRA/) 48 | (a package for comparing results of "binary classification and ranking methods" 49 | with a Shiny web application for interactive analyses), 50 | [rnaseqcomp](http://bioconductor.org/packages/rnaseqcomp) (a package for 51 | comparing results of RNA-seq quantification pipelines), and 52 | [dsc](https://github.com/stephenslab/dsc) (a framework for "managing 53 | computational benchmarking experiments that compare several competing methods" 54 | written in Python but capable of running methods implemented in both Python and R). 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /tests/testthat/test-updateBench.R: -------------------------------------------------------------------------------- 1 | library(SummarizedBenchmark) 2 | context("updateBench") 3 | 4 | ## sample t-test data set 5 | data(tdat) 6 | 7 | test_that("basic updateBench call works", { 8 | ## basic BenchDesign 9 | bd <- BenchDesign(data = tdat) 10 | BDMethod(bd, "bonf") <- BDMethod(x = p.adjust, 11 | params = rlang::quos(p = pval, method = "bonferroni"), 12 | meta = list(purpose = "for testing")) 13 | BDMethod(bd, "bh") <- BDMethod(x = p.adjust, 14 | params = rlang::quos(p = pval, method = "BH"), 15 | meta = list(purpose = "for comparing")) 16 | sb <- buildBench(bd) 17 | 18 | ## basic call w/ just SummarizedBenchmark 19 | bdcomp <- compareBenchDesigns(sb) 20 | expect_output(bdupdate <- updateBench(sb), "Update SummarizedBenchmark \\(dryrun\\)") 21 | expect_equal(bdcomp$methods$res, bdupdate$methods$res) 22 | expect_equal(dplyr::select(bdcomp$methods[[2]], -post, -f), 23 | dplyr::select(bdupdate$methods[[2]], -post, -f)) 24 | expect_equal(dplyr::select(bdcomp$methods[[3]], -post, -f), 25 | dplyr::select(bdupdate$methods[[3]], -post, -f)) 26 | expect_equal(bdcomp$data, bdupdate$data) 27 | expect_equal(bdcomp$methods$res$overlap, rep("Both", 2)) 28 | expect_true(all(unlist(dplyr::select_if(bdcomp$methods$res, is.logical)))) 29 | 30 | expect_error(sb2 <- updateBench(sb, dryrun = FALSE), "MD5 hash") 31 | expect_silent(sb2 <- updateBench(sb, dryrun = FALSE, data = tdat)) 32 | expect_identical(sb, sb2) 33 | 34 | ## basic call w/ new BenchDesign 35 | bd2 <- bd 36 | BDMethod(bd2, "holm") <- BDMethod(x = p.adjust, params = rlang::quos(p = pval, method = "holm")) 37 | BDMethod(bd2, "bh") <- NULL 38 | 39 | expect_silent(bdcomp <- compareBenchDesigns(sb, bd2)) 40 | expect_output(bdupdate <- updateBench(sb, bd2), "Update SummarizedBenchmark \\(dryrun\\)") 41 | expect_output(updateBench(sb, bd2, keepAll = FALSE), "Drop") 42 | expect_equal(bdcomp$methods$res, bdupdate$methods$res) 43 | expect_equal(dplyr::filter(bdupdate$methods$res, label == "holm")$overlap, "yOnly") 44 | expect_equal(dplyr::filter(bdupdate$methods$res, label == "bh")$overlap, "xOnly") 45 | expect_equal(dplyr::filter(bdupdate$methods$res, label == "bonf")$overlap, "Both") 46 | expect_true(all(is.na(unlist(dplyr::select_if(dplyr::filter(bdcomp$methods$res, label == "holm"), is.logical))))) 47 | 48 | expect_silent(sb2 <- updateBench(sb, bd2, dryrun = FALSE, data = tdat)) 49 | expect_silent(sb2drop <- updateBench(sb, bd2, dryrun = FALSE, keepAll = FALSE, data = tdat)) 50 | 51 | expect_equal(ncol(sb2), 3L) 52 | expect_equal(ncol(sb2drop), 2L) 53 | expect_equal(sort(colnames(sb2)), c("bh", "bonf", "holm")) 54 | expect_equal(sort(colnames(sb2drop)), c("bonf", "holm")) 55 | 56 | expect_length(BDMethodList(sb2), 3L) 57 | expect_length(BDMethodList(sb2drop), 2L) 58 | 59 | expect_equal(length(metadata(sb2)$sessions), 2L) 60 | expect_equal(length(metadata(sb2drop)$sessions), 2L) 61 | 62 | expect_equal(sort(metadata(sb2)$sessions[[1]]$methods), c("bh", "bonf")) 63 | expect_equal(metadata(sb2)$sessions[[2]]$methods, "holm") 64 | 65 | expect_equal(metadata(sb2drop)$sessions[[1]]$methods, "bonf") 66 | expect_equal(metadata(sb2drop)$sessions[[2]]$methods, "holm") 67 | expect_equal(names(metadata(sb2drop)$sessions[[1]]$results), "bonf") 68 | expect_equal(names(metadata(sb2drop)$sessions[[2]]$results), "holm") 69 | }) 70 | -------------------------------------------------------------------------------- /R/BDData-constructor.R: -------------------------------------------------------------------------------- 1 | #' Create a new BDData object 2 | #' 3 | #' @description 4 | #' Initializes a new BDData object of benchmarking data. 5 | #' 6 | #' Data sets are stored as BDData objects within BenchDesign 7 | #' objects as well as SummarizedBenchmark objects. However, because 8 | #' data is directly specified to the BenchDesign constructor, there 9 | #' is usually no need to call the BDData constructor to create completely 10 | #' new data objects. 11 | #' 12 | #' The BDData constructor is most useful for extracting the data sets 13 | #' contained in BenchDesign objects as well as SummarizedBenchmark objects. 14 | #' By default, the BDData object stored in SummarizedBenchmark objects will 15 | #' be MD5 hashes rather than the complete original data set. 16 | #' \code{\link{compareBDData}} can be used to compare various forms of 17 | #' BDData, as shown in the examples below. 18 | #' 19 | #' @param data a list object of data or MD5 hash string 20 | #' 21 | #' @return 22 | #' BDData object 23 | #' 24 | #' @examples 25 | #' ## construct from data.frame 26 | #' datadf <- data.frame(x = 1:5, y = runif(5)) 27 | #' bdd_df <- BDData(datadf) 28 | #' bdd_df 29 | #' 30 | #' ## construct from MD5 hash of data.frame 31 | #' bdd_md5 <- BDData(digest::digest(datadf)) 32 | #' bdd_md5 33 | #' 34 | #' ## compare two BDData objects 35 | #' compareBDData(bdd_df, bdd_md5) 36 | #' 37 | #' ## note that the data is the same, i.e. the MD5 hashes match, but the 38 | #' ## data types ("data" vs. "md5has") are different 39 | #' 40 | #' @seealso \code{\link{BDData-class}}, \code{\link{BenchDesign}} 41 | #' @name BDData 42 | #' @importFrom methods .valueClassTest 43 | #' @export 44 | #' @author Patrick Kimes 45 | NULL 46 | 47 | .BDData.default <- function(data) { 48 | if (is(data, "character")) { 49 | new("BDData", data = data, type = "md5hash") 50 | } else { 51 | new("BDData", data = data, type = "data") 52 | } 53 | } 54 | 55 | .BDData.bd <- function(data) { 56 | data@data 57 | } 58 | 59 | .BDData.sb <- function(data) { 60 | BDData(BenchDesign(data)) 61 | } 62 | 63 | #' @rdname BDData 64 | setMethod("BDData", signature(data = "ANY"), .BDData.default) 65 | 66 | #' @rdname BDData 67 | setMethod("BDData", signature(data = "BenchDesign"), .BDData.bd) 68 | 69 | #' @rdname BDData 70 | setMethod("BDData", signature(data = "SummarizedBenchmark"), .BDData.sb) 71 | 72 | #' @rdname BDData 73 | setMethod("BDData", signature(data = "BDData"), function(data) { data }) 74 | 75 | 76 | #' Hash data in BDData object 77 | #' 78 | #' @description 79 | #' Repliaces data stored in a \code{\link[=BDData-class]{BDData}} object 80 | #' with the MD5 hash of the data. If the data was already a MD5 hash, the 81 | #' original object is returned unchanged. The method can be called directly 82 | #' on \code{\link[=BenchDesign-class]{BenchDesign}} objects to hash the 83 | #' underlying data as well. 84 | #' 85 | #' @param object a \code{BDData} or \code{BenchDesign} object 86 | #' 87 | #' @return 88 | #' an object of the same class as \code{object} with data 89 | #' converted to a MD5 hash. 90 | #' 91 | #' @name hashBDData 92 | #' @importFrom digest digest 93 | #' @export 94 | #' @author Patrick Kimes 95 | NULL 96 | 97 | .hashBDData <- function(object) { 98 | if (object@type == "md5hash") 99 | return(object) 100 | BDData(digest::digest(object@data, algo = "md5")) 101 | } 102 | 103 | .hashBDData.bd <- function(object) { 104 | BDData(object) <- hashBDData(BDData(object)) 105 | object 106 | } 107 | 108 | #' @rdname hashBDData 109 | setMethod("hashBDData", signature(object = "BDData"), .hashBDData) 110 | 111 | #' @rdname hashBDData 112 | setMethod("hashBDData", signature(object = "BenchDesign"), .hashBDData.bd) 113 | 114 | 115 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | S3method(addMethod,BenchDesign) 4 | S3method(dropMethod,BenchDesign) 5 | S3method(expandMethod,BenchDesign) 6 | S3method(modifyMethod,BenchDesign) 7 | export("BDData<-") 8 | export("BDMethod<-") 9 | export("BDMethodList<-") 10 | export("groundTruths<-") 11 | export("performanceMetrics<-") 12 | export(BDData) 13 | export(BDMethod) 14 | export(BDMethodList) 15 | export(BenchDesign) 16 | export(SummarizedBenchmark) 17 | export(addMethod) 18 | export(addPerformanceMetric) 19 | export(availableMetrics) 20 | export(buildBench) 21 | export(compareBDData) 22 | export(compareBDMethod) 23 | export(compareBenchDesigns) 24 | export(dropMethod) 25 | export(estimateMetricsForAssay) 26 | export(estimatePerformanceMetrics) 27 | export(expandMethod) 28 | export(groundTruths) 29 | export(hashBDData) 30 | export(modifyMethod) 31 | export(performanceMetrics) 32 | export(plotMethodsOverlap) 33 | export(plotROC) 34 | export(printMethod) 35 | export(printMethods) 36 | export(tidyBDMethod) 37 | export(tidyUpMetrics) 38 | export(updateBench) 39 | exportClasses(BDData) 40 | exportClasses(BDMethod) 41 | exportClasses(BDMethodList) 42 | exportClasses(BenchDesign) 43 | exportClasses(SummarizedBenchmark) 44 | exportMethods("BDData<-") 45 | exportMethods("BDMethod") 46 | exportMethods("BDMethod<-") 47 | exportMethods("BDMethodList") 48 | exportMethods("BDMethodList<-") 49 | exportMethods("BenchDesign") 50 | exportMethods("assayNames<-") 51 | exportMethods("groundTruths") 52 | exportMethods("groundTruths<-") 53 | exportMethods("mcols<-") 54 | exportMethods("performanceMetrics") 55 | exportMethods("performanceMetrics<-") 56 | exportMethods(BDMethod) 57 | exportMethods(compareBenchDesigns) 58 | exportMethods(tidyBDMethod) 59 | import(BiocParallel) 60 | import(SummarizedExperiment) 61 | import(dplyr) 62 | importClassesFrom(S4Vectors,SimpleList) 63 | importFrom(BiocGenerics,sd) 64 | importFrom(S4Vectors,DataFrame) 65 | importFrom(S4Vectors,SimpleList) 66 | importFrom(S4Vectors,cor) 67 | importFrom(S4Vectors,elementMetadata) 68 | importFrom(S4Vectors,metadata) 69 | importFrom(S4Vectors,params) 70 | importFrom(SummarizedExperiment,cbind) 71 | importFrom(UpSetR,upset) 72 | importFrom(crayon,bold) 73 | importFrom(crayon,green) 74 | importFrom(crayon,red) 75 | importFrom(crayon,yellow) 76 | importFrom(digest,digest) 77 | importFrom(dplyr,as_tibble) 78 | importFrom(dplyr,left_join) 79 | importFrom(ggplot2,aes) 80 | importFrom(ggplot2,geom_abline) 81 | importFrom(ggplot2,geom_line) 82 | importFrom(ggplot2,ggplot) 83 | importFrom(ggplot2,xlim) 84 | importFrom(ggplot2,ylim) 85 | importFrom(mclust,adjustedRandIndex) 86 | importFrom(methods,.valueClassTest) 87 | importFrom(methods,as) 88 | importFrom(methods,formalArgs) 89 | importFrom(methods,is) 90 | importFrom(methods,new) 91 | importFrom(methods,slot) 92 | importFrom(methods,validObject) 93 | importFrom(rlang,"!!!") 94 | importFrom(rlang,"!!") 95 | importFrom(rlang,":=") 96 | importFrom(rlang,enquo) 97 | importFrom(rlang,eval_tidy) 98 | importFrom(rlang,flatten) 99 | importFrom(rlang,get_expr) 100 | importFrom(rlang,is_quosure) 101 | importFrom(rlang,is_quosures) 102 | importFrom(rlang,quo) 103 | importFrom(rlang,quo_text) 104 | importFrom(rlang,quos) 105 | importFrom(sessioninfo,session_info) 106 | importFrom(stringr,str_pad) 107 | importFrom(stringr,str_trunc) 108 | importFrom(tibble,is_tibble) 109 | importFrom(tidyr,gather) 110 | importFrom(tidyr,spread) 111 | importFrom(tidyr,unnest) 112 | importFrom(utils,head) 113 | importFrom(utils,packageName) 114 | importFrom(utils,packageVersion) 115 | importMethodsFrom(S4Vectors,"elementMetadata<-") 116 | importMethodsFrom(S4Vectors,"mcols<-") 117 | importMethodsFrom(S4Vectors,elementMetadata) 118 | importMethodsFrom(S4Vectors,mcols) 119 | importMethodsFrom(SummarizedExperiment,"assayNames<-") 120 | importMethodsFrom(SummarizedExperiment,assayNames) 121 | -------------------------------------------------------------------------------- /docs/pkgdown.js: -------------------------------------------------------------------------------- 1 | /* http://gregfranko.com/blog/jquery-best-practices/ */ 2 | (function($) { 3 | $(function() { 4 | 5 | $("#sidebar") 6 | .stick_in_parent({offset_top: 40}) 7 | .on('sticky_kit:bottom', function(e) { 8 | $(this).parent().css('position', 'static'); 9 | }) 10 | .on('sticky_kit:unbottom', function(e) { 11 | $(this).parent().css('position', 'relative'); 12 | }); 13 | 14 | $('body').scrollspy({ 15 | target: '#sidebar', 16 | offset: 60 17 | }); 18 | 19 | $('[data-toggle="tooltip"]').tooltip(); 20 | 21 | var cur_path = paths(location.pathname); 22 | var links = $("#navbar ul li a"); 23 | var max_length = -1; 24 | var pos = -1; 25 | for (var i = 0; i < links.length; i++) { 26 | if (links[i].getAttribute("href") === "#") 27 | continue; 28 | // Ignore external links 29 | if (links[i].host !== location.host) 30 | continue; 31 | 32 | var nav_path = paths(links[i].pathname); 33 | 34 | var length = prefix_length(nav_path, cur_path); 35 | if (length > max_length) { 36 | max_length = length; 37 | pos = i; 38 | } 39 | } 40 | 41 | // Add class to parent
  • , and enclosing
  • if in dropdown 42 | if (pos >= 0) { 43 | var menu_anchor = $(links[pos]); 44 | menu_anchor.parent().addClass("active"); 45 | menu_anchor.closest("li.dropdown").addClass("active"); 46 | } 47 | }); 48 | 49 | function paths(pathname) { 50 | var pieces = pathname.split("/"); 51 | pieces.shift(); // always starts with / 52 | 53 | var end = pieces[pieces.length - 1]; 54 | if (end === "index.html" || end === "") 55 | pieces.pop(); 56 | return(pieces); 57 | } 58 | 59 | // Returns -1 if not found 60 | function prefix_length(needle, haystack) { 61 | if (needle.length > haystack.length) 62 | return(-1); 63 | 64 | // Special case for length-0 haystack, since for loop won't run 65 | if (haystack.length === 0) { 66 | return(needle.length === 0 ? 0 : -1); 67 | } 68 | 69 | for (var i = 0; i < haystack.length; i++) { 70 | if (needle[i] != haystack[i]) 71 | return(i); 72 | } 73 | 74 | return(haystack.length); 75 | } 76 | 77 | /* Clipboard --------------------------*/ 78 | 79 | function changeTooltipMessage(element, msg) { 80 | var tooltipOriginalTitle=element.getAttribute('data-original-title'); 81 | element.setAttribute('data-original-title', msg); 82 | $(element).tooltip('show'); 83 | element.setAttribute('data-original-title', tooltipOriginalTitle); 84 | } 85 | 86 | if(ClipboardJS.isSupported()) { 87 | $(document).ready(function() { 88 | var copyButton = ""; 89 | 90 | $(".examples, div.sourceCode").addClass("hasCopyButton"); 91 | 92 | // Insert copy buttons: 93 | $(copyButton).prependTo(".hasCopyButton"); 94 | 95 | // Initialize tooltips: 96 | $('.btn-copy-ex').tooltip({container: 'body'}); 97 | 98 | // Initialize clipboard: 99 | var clipboardBtnCopies = new ClipboardJS('[data-clipboard-copy]', { 100 | text: function(trigger) { 101 | return trigger.parentNode.textContent; 102 | } 103 | }); 104 | 105 | clipboardBtnCopies.on('success', function(e) { 106 | changeTooltipMessage(e.trigger, 'Copied!'); 107 | e.clearSelection(); 108 | }); 109 | 110 | clipboardBtnCopies.on('error', function() { 111 | changeTooltipMessage(e.trigger,'Press Ctrl+C or Command+C to copy'); 112 | }); 113 | }); 114 | } 115 | })(window.jQuery || window.$) 116 | -------------------------------------------------------------------------------- /tests/testthat/test-BDMethod.R: -------------------------------------------------------------------------------- 1 | library(SummarizedBenchmark) 2 | context("BDMethod and BDMethodList") 3 | 4 | ## sample SummarizedBenchmark object 5 | data(quantSB) 6 | 7 | test_that("BDMethod constructor accepts all valid input", { 8 | ## simplest call 9 | expect_silent(bdm <- BDMethod(x = base::identity)) 10 | expect_is(bdm, "BDMethod") 11 | expect_equal(rlang::expr_text(bdm@fc), "base::identity") 12 | expect_identical(bdm@f, base::identity) 13 | expect_true(rlang::is_quosures(bdm@params)) 14 | expect_null(bdm@post) 15 | expect_null(bdm@meta) 16 | 17 | ## call with all params specified 18 | expect_silent(bdm <- BDMethod(x = base::identity, 19 | params = rlang::quos(x = y), 20 | post = list(p1 = base::identity, 21 | p2 = base::identity), 22 | meta = list(m1 = "a"))) 23 | expect_is(bdm, "BDMethod") 24 | expect_identical(bdm@f, base::identity) 25 | expect_true(rlang::is_quosures(bdm@params)) 26 | expect_is(bdm@post, "list") 27 | expect_is(bdm@meta,"list") 28 | 29 | ## call with quoted function 30 | expect_silent(bdm <- BDMethod(x = rlang::quo(base::identity))) 31 | expect_is(bdm, "BDMethod") 32 | expect_identical(bdm@f, base::identity) 33 | expect_equal(rlang::expr_text(bdm@fc), "base::identity") 34 | 35 | ## call with BenchDesign 36 | expect_silent(bdm <- BDMethod(BenchDesign(allSB[[1]]), i = 2)) 37 | expect_is(bdm, "BDMethod") 38 | }) 39 | 40 | test_that("BDMethodList constructor accepts all valid input", { 41 | bdm1 <- BDMethod(x = base::identity) 42 | bdm2 <- BDMethod(x = base::sqrt) 43 | bdm3 <- BDMethod(x = base::exp) 44 | 45 | bd1 <- BenchDesign(m1 = bdm1) 46 | bd2 <- BenchDesign(m2 = bdm2, m3 = bdm3) 47 | 48 | ## simplest call 49 | expect_silent(bdml1 <- BDMethodList(m1 = bdm1)) 50 | expect_silent(bdml2 <- BDMethodList(m2 = bdm2, m3 = bdm3)) 51 | expect_is(bdml1, "BDMethodList") 52 | expect_is(bdml2, "BDMethodList") 53 | expect_identical(bdml1[[1]], bdm1) 54 | expect_identical(bdml2[[2]], bdm3) 55 | 56 | ## call with BenchDesign 57 | expect_silent(bd1_bdml <- BDMethodList(bd1)) 58 | expect_identical(bd1_bdml, bdml1) 59 | expect_silent(bd2_bdml <- BDMethodList(bd2)) 60 | expect_identical(bd2_bdml, bdml2) 61 | 62 | ## call with SummarizedBenchmark) 63 | expect_silent(sb_bdml <- BDMethodList(allSB[[1]])) 64 | expect_is(sb_bdml, "BDMethodList") 65 | 66 | ## combine bdm and bdml 67 | expect_is(BDMethodList(m1 = bdm1, bdml2), "BDMethodList") 68 | expect_length(BDMethodList(m1 = bdm1, bdml2), 3) 69 | 70 | ## combine bdm and bd 71 | expect_is(BDMethodList(m2 = bdm2, bd1), "BDMethodList") 72 | expect_length(BDMethodList(m2 = bdm2, bd1), 2) 73 | 74 | ## combine bdml and bdml 75 | expect_is(BDMethodList(bdml1, bdml2), "BDMethodList") 76 | expect_length(BDMethodList(bdml1, bdml2), 3) 77 | 78 | ## combine bdml and bd 79 | expect_is(BDMethodList(bd1, bdml2), "BDMethodList") 80 | expect_length(BDMethodList(bd1, bdml2), 3) 81 | 82 | ## combine bd and bd 83 | expect_is(BDMethodList(bd1, bd2), "BDMethodList") 84 | expect_identical(BDMethodList(bd1, bd2), BDMethodList(bd1_bdml, bd2_bdml)) 85 | }) 86 | 87 | test_that("BDMethod/List setters work", { 88 | bd <- BenchDesign(allSB[[1]]) 89 | bdm1 <- BDMethod(x = base::identity) 90 | bdml <- BDMethodList(m1 = bdm1) 91 | 92 | ## set bdml bdm 93 | expect_silent(BDMethod(bdml, "newMethod") <- bdm1) 94 | expect_identical(BDMethod(bdml, i = "newMethod"), bdm1) 95 | expect_length(bdml, 2) 96 | 97 | ## set benchdesign bdm 98 | expect_silent(BDMethod(bd, "newMethod") <- bdm1) 99 | expect_identical(BDMethod(bd, i = "newMethod"), bdm1) 100 | 101 | ## set benchdesign bdml 102 | expect_silent(BDMethodList(bd) <- bdml) 103 | expect_identical(BDMethodList(bd), bdml) 104 | }) 105 | -------------------------------------------------------------------------------- /R/BDMethodList-constructor.R: -------------------------------------------------------------------------------- 1 | #' Create a new BDMethodList object 2 | #' 3 | #' @description 4 | #' Initializes a new SimpleList of BenchDesign method (\code{\link[=BDMethod-class]{BDMethod}}) objects. 5 | #' 6 | #' Similar to \code{\link[=BDMethod-class]{BDMethod}} objects, \code{\link[=BDMethodList-class]{BDMethodList}} 7 | #' typically do not need to be 8 | #' directly constructed. Because the list of methods is only useful as art of a 9 | #' \code{\link[=BenchDesign-class]{BenchDesign}} object, it is more common to simply manipulate the 10 | #' list of methods through calls to the corresponding \code{\link[=BenchDesign-class]{BenchDesign}}, e.g. 11 | #' \code{\link{addMethod}} to add a new method to the list. 12 | #' 13 | #' The constructor can also be used to access the \code{\link[=BDMethodList-class]{BDMethodList}} list 14 | #' of methods in a \code{\link[=BenchDesign-class]{BenchDesign}} object. 15 | #' 16 | #' @param ... a named list of \code{\link[=BDMethod-class]{BDMethod}} objects 17 | #' @param x a \code{\link[=BenchDesign-class]{BenchDesign}} or 18 | #' \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object to extract 19 | #' the list of methods from. (default = NULL) 20 | #' 21 | #' @return 22 | #' BDMethodList object 23 | #' 24 | #' @examples 25 | #' ## construct an empty list 26 | #' bdml <- BDMethodList() 27 | #' 28 | #' ## construct a list with BDMethod objects 29 | #' bdml <- BDMethodList(m_method = BDMethod(base::mean), 30 | #' s_method = BDMethod(function(x) { x^2 })) 31 | #' bdml 32 | #' 33 | #' ## construct a BenchDesign with a BDMethodList 34 | #' bd <- BenchDesign(methods = bdml) 35 | #' 36 | #' ## access the BDMethodList in the BenchDesign 37 | #' BDMethodList(bd) 38 | #' 39 | #' @seealso \code{\link{BDMethodList-class}}, \code{\link{BenchDesign}}, \code{\link{BDMethod}} 40 | #' @name BDMethodList 41 | #' @importFrom rlang flatten 42 | #' @importFrom S4Vectors SimpleList 43 | #' @importFrom methods .valueClassTest 44 | #' @export 45 | #' @author Patrick Kimes 46 | NULL 47 | 48 | .BDMethodList <- function(..., x) { 49 | ml <- list(...) 50 | if (!is.null(x)) 51 | ml <- c(ml, x) 52 | if (length(ml) == 0) 53 | ml <- list() 54 | 55 | ## allow shortcut for calling on list, SB, BD w/out specifying x= 56 | if (length(ml) == 1) { 57 | if (is.list(ml[[1]]) || is(ml[[1]], "List")) { 58 | ml <- ml[[1]] 59 | } else if (is(ml[[1]], "BenchDesign") || is(ml[[1]], "SummarizedBenchmark")) { 60 | return(BDMethodList(x = ml[[1]])) 61 | } 62 | } 63 | 64 | ## extract any BD objects and flatten any BDML objects 65 | ml_is_bd <- unlist(lapply(ml, is, "BenchDesign")) 66 | ml_is_bdm <- unlist(lapply(ml, is, "BDMethod")) 67 | ml_is_list <- unlist(lapply(ml, is, "list")) | unlist(lapply(ml, is, "List")) 68 | if (any(!(ml_is_bd | ml_is_bdm | ml_is_list))) 69 | stop("When specifying more than one object, only specify BenchDesign, BDMethod or list/List objects.") 70 | 71 | if (any(ml_is_bd)) { 72 | ml[ml_is_bd] <- lapply(ml[ml_is_bd], BDMethodList) 73 | } 74 | if (any(ml_is_bd | ml_is_list)) { 75 | ml[ml_is_bd | ml_is_list] <- lapply(ml[ml_is_bd | ml_is_list], as.list) 76 | ml <- rlang::flatten(ml) 77 | } 78 | 79 | if (length(ml) > 0 && (is.null(names(ml)) || 80 | any(!nzchar(names(ml))) || 81 | any(duplicated(names(ml))))) 82 | stop("Methods must be specified with unique names.") 83 | 84 | if (is(ml, "List")) 85 | return(new("BDMethodList", ml)) 86 | else 87 | return(new("BDMethodList", S4Vectors::SimpleList(ml))) 88 | } 89 | 90 | #' @rdname BDMethodList 91 | setMethod("BDMethodList", signature(x = "ANY"), .BDMethodList) 92 | 93 | #' @rdname BDMethodList 94 | #' @name coerce 95 | setAs("list", "BDMethodList", function(from) { 96 | new("BDMethodList", as(from, "SimpleList")) 97 | }) 98 | setAs("List", "BDMethodList", function(from) { 99 | new("BDMethodList", as(from, "SimpleList")) 100 | }) 101 | 102 | -------------------------------------------------------------------------------- /R/modifyMethod.R: -------------------------------------------------------------------------------- 1 | #' Modify method in BenchDesign object 2 | #' 3 | #' @description 4 | #' Takes a \code{\link[=BenchDesign-class]{BenchDesign}} object, the name of an 5 | #' existing method, and new parameter specifications, 6 | #' and returns a modified \code{\link[=BenchDesign-class]{BenchDesign}} object with 7 | #' the specified changes. 8 | #' 9 | #' @param bd \code{\link[=BenchDesign-class]{BenchDesign}} object. 10 | #' @param label Character name of method to be modified. 11 | #' @param params Named quosure list created using \code{\link[rlang:quotation]{quos}} of 12 | #' `parameter = value` paiars to replace in the method definition. 13 | #' The `post`, and `meta` parameters of the method can be 14 | #' modified using the special keywords, `bd.post`, and `bd.meta` 15 | #' (the prefix denoting that these values should modify \code{\link[=BenchDesign-class]{BenchDesign}} 16 | #' parameters). All other named parameters will be added to the list of 17 | #' parameters to be passed to `func`. 18 | #' @param .overwrite Logical whether to overwrite the complete existing list of 19 | #' parameters to be passed to `func` (\code{TRUE}), or to simply add 20 | #' the new parameters to the existing list and only replace overlapping 21 | #' parameters (\code{FALSE}). (default = \code{FALSE}) 22 | #' 23 | #' @examples 24 | #' ## empty BenchDesign 25 | #' bench <- BenchDesign() 26 | #' 27 | #' ## add method 28 | #' bench <- addMethod(bench, label = "qv", 29 | #' func = qvalue::qvalue, 30 | #' post = function(x) { x$qvalue }, 31 | #' meta = list(note = "storey's q-value"), 32 | #' params = rlang::quos(p = pval)) 33 | #' 34 | #' ## modify method 'meta' property of 'qv' method 35 | #' bench <- modifyMethod(bench, label = "qv", 36 | #' params = rlang::quos(bd.meta = 37 | #' list(note = "Storey's q-value"))) 38 | #' 39 | #' ## verify that method has been updated 40 | #' printMethod(bench, "qv") 41 | #' 42 | #' @return 43 | #' Modified \code{\link[=BenchDesign-class]{BenchDesign}} object with single method 44 | #' parameters modified. 45 | #' 46 | #' @seealso \code{\link{addMethod}}, \code{\link{expandMethod}}, \code{\link{dropMethod}} 47 | #' @md 48 | #' @importFrom rlang is_quosures eval_tidy 49 | #' @export 50 | #' @author Patrick Kimes 51 | modifyMethod <- function(bd, label, params, .overwrite = FALSE) { 52 | UseMethod("modifyMethod") 53 | } 54 | 55 | #' @export 56 | modifyMethod.BenchDesign <- function(bd, label, params, .overwrite = FALSE) { 57 | ## verify that method definition already exists 58 | if (!(label %in% names(bd@methods))) { 59 | stop("Specified method is not defined in BenchDesign.") 60 | } 61 | 62 | if (!rlang::is_quosures(params)) { 63 | stop("Please supply 'func' parameters to 'params =' as ", 64 | "a list of quosures using rlang::quos.\n", 65 | "e.g. params = quos(param1 = x, param2 = y)") 66 | } 67 | 68 | ## modify and add to bench 69 | bm <- bd@methods[[label]] 70 | bd@methods[[label]] <- .modmethod(bm, params, .overwrite) 71 | 72 | return(bd) 73 | } 74 | 75 | 76 | #' Modify BenchDesign Method 77 | #' 78 | #' Given a method defined in a BenchDesign, this helper function 79 | #' returns a modified method with new parameters defined as a 80 | #' list of quosures. 81 | #' 82 | #' @param m method 83 | #' @param q quosure list of new parameters 84 | #' @param .overwrite logical whether to overwrite parameters 85 | #' 86 | #' @return 87 | #' modified method. 88 | #' 89 | #' @rdname modmethod 90 | #' @keywords internal 91 | #' @author Patrick Kimes 92 | .modmethod <- function(m, q, .overwrite) { 93 | ## parse out post, meta 94 | if ("bd.post" %in% names(q)) { 95 | new_post <- rlang::eval_tidy(q$bd.post) 96 | } else { 97 | new_post <- m@post 98 | } 99 | if ("bd.meta" %in% names(q)) { 100 | new_meta <- rlang::eval_tidy(q$bd.meta) 101 | } else { 102 | new_meta <- m@meta 103 | } 104 | 105 | ## process named parameters to be used for func 106 | q <- q[! names(q) %in% c("bd.post", "bd.meta")] 107 | if (.overwrite) { 108 | new_params <- q 109 | } else { 110 | new_params <- replace(m@params, names(q), q) 111 | } 112 | 113 | ## easiest way to create modified BDMethod w/ m@f, m@fc 114 | bd <- BDMethod(x = m@f, post = new_post, meta = new_meta, 115 | params = new_params) 116 | bd@fc <- m@fc 117 | bd 118 | } 119 | -------------------------------------------------------------------------------- /man/addMethod.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/addMethod.R 3 | \name{addMethod} 4 | \alias{addMethod} 5 | \title{Add method to BenchDesign object} 6 | \usage{ 7 | addMethod(bd, label, func, params = rlang::quos(), post = NULL, 8 | meta = NULL) 9 | } 10 | \arguments{ 11 | \item{bd}{\code{\link[=BenchDesign-class]{BenchDesign}} object.} 12 | 13 | \item{label}{Character name for the method.} 14 | 15 | \item{func}{Primary function to be benchmarked.} 16 | 17 | \item{params}{Named quosure list created using \code{\link[rlang:quotation]{quos}} of 18 | \code{parameter = value} pairs to be passed to \code{func}.} 19 | 20 | \item{post}{Optional post-processing function that takes 21 | results of \code{func} as input. Ignored if \code{NULL}. 22 | If multiple assays (metrics) should be generated for each 23 | method, this can be accomplished by specifying a named 24 | list of post-processing functions, one for each assay. 25 | (default = \code{NULL})} 26 | 27 | \item{meta}{Optional metadata information for method to be 28 | included in \code{colData} of \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object 29 | generated using \code{link{buildBench}}. See Details for more 30 | information. Ignored if \code{NULL}. (default = \code{NULL})} 31 | } 32 | \value{ 33 | Modified \code{\link[=BenchDesign-class]{BenchDesign}} object with 34 | new method added. 35 | } 36 | \description{ 37 | Takes a \code{\link[=BenchDesign-class]{BenchDesign}} object 38 | and the definition of a new method for benchmarking and returns 39 | the original \code{\link[=BenchDesign-class]{BenchDesign}} with 40 | the new method included. 41 | 42 | At a minimum, a method label (\code{label =}), and the 43 | workhorse function for the method (\code{func =}) must be specified 44 | for the new method. 45 | 46 | Parameters for the method must be specified as a 47 | \code{\link[rlang:quotation]{quos}} named list of \code{parameter = value} pairs 48 | mapping entries in the benchmarking data to the function parameters. 49 | For users familiar with the \pkg{ggplot2} package, this can be 50 | viewed similar to the \code{aes =} mapping of data to geometry 51 | parameters. 52 | 53 | An optional secondary function, \code{post}, can be specified if 54 | the output of the workhorse function, \code{func}, needs to be 55 | further processed. As an example, \code{post} may be a simple 56 | "getter" function for accessing the column of interest from 57 | the large object returned by \code{func}. 58 | } 59 | \details{ 60 | The optional \code{meta} parameter accepts a named list of metadata 61 | tags to be included for the method in the resulting 62 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 63 | object. This can be useful for two primary cases. First, it can help keep 64 | analyses better organized by allowing the specification of additional 65 | information that should be stored with methods, e.g. a tag for "method type" 66 | or descriptive information on why the method was included in the comparison. 67 | Second, and more improtantly, the \code{meta} parameter can be used to overwrite 68 | the package and version information that is automatically extracted from the 69 | function specified to \code{func}. This is particularly useful when the function 70 | passed to \code{func} is a wrapper for a script in (or outside of) R, and the 71 | appropriate package and version information can't be directly pulled from 72 | \code{func}. In this case, the user can either manually specify the \code{"pkg_name"} 73 | and \code{"pkg_vers"} values to \code{meta} as a list, or specify a separate function 74 | that should be used to determine the package name and version. If a separate 75 | function should be used, it should be passed to \code{meta} as a list entry 76 | with the name \code{pkg_func} and first quoted using \code{\link[rlang:quotation]{quo}}, e.g. 77 | \code{list(pkg_func = quo(p.adjust))}. 78 | } 79 | \examples{ 80 | ## create example data set of p-values 81 | df <- data.frame(pval = runif(100)) 82 | 83 | ## example calculating qvalue from pvalues 84 | 85 | ## using standard call 86 | qv <- qvalue::qvalue(p = df$pval) 87 | qv <- qv$qvalue 88 | 89 | ## adding same method to BenchDesign 90 | bench <- BenchDesign(data = df) 91 | bench <- addMethod(bench, 92 | label = "qv", 93 | func = qvalue::qvalue, 94 | post = function(x) { x$qvalue }, 95 | params = rlang::quos(p = pval)) 96 | 97 | } 98 | \seealso{ 99 | \code{\link{modifyMethod}}, \code{\link{expandMethod}}, \code{\link{dropMethod}} 100 | } 101 | \author{ 102 | Patrick Kimes 103 | } 104 | -------------------------------------------------------------------------------- /R/SummarizedBenckmark-constructor.R: -------------------------------------------------------------------------------- 1 | #' @title Create a new SummarizedBenchmark object 2 | #' @aliases SummarizedBenchmark 3 | #' @description 4 | #' Function to construct \code{SummarizedBenchmark} objects. 5 | #' 6 | #' @param assays A list containing outputs of the methods to be 7 | #' benchmark. Each element of the list must contain a matrix or 8 | #' data.frame of n x m, n being the number of features tested 9 | #' (e.g. genes) and m being the number of methods in the benchmark. 10 | #' Each element of the list must contain a single assay 11 | #' (the outputs of the methods). For example, 12 | #' for a benchmark of differential expression methods, one assay could contain 13 | #' the q-values from the different methods and another assay could be the 14 | #' estimated log fold changes. 15 | #' @param colData A \code{\link{DataFrame}} describing the annotation of the 16 | #' methods. These could include version of the software or the parameters 17 | #' used to run them. 18 | #' @param ftData A \code{\link{DataFrame}} object describing the rows. This parameter 19 | #' is equivalent to the parameter rowData of a SummarizedExperiment. 20 | #' @param groundTruth If present, a \code{\link{DataFrame}} containing the ground truths. 21 | #' If provided, the number of columns must be the same as the number of assays 22 | #' (NA's are accepted). 23 | #' The names of the columns should have the same names as the assays. 24 | #' @param performanceMetrics A \code{\link{SimpleList}} of the same 25 | #' length as the number of assays. Each element of the list must be a list 26 | #' of functions. Each function must contain the parameters 'query' and 27 | #' 'truth'. 28 | #' @param BenchDesign A \code{\link{BenchDesign}} containing the code used 29 | #' to construct the object. (default = NULL) 30 | #' @param ... Additional parameters passed to \code{\link{SummarizedExperiment}}. 31 | #' 32 | #' @author Alejandro Reyes 33 | #' 34 | #' @examples 35 | #' 36 | #' ## loading the example data from iCOBRA 37 | #' library(iCOBRA) 38 | #' data(cobradata_example) 39 | #' 40 | #' ## a bit of data wrangling and reformatting 41 | #' assays <- list( 42 | #' qvalue=cobradata_example@padj, 43 | #' logFC=cobradata_example@score ) 44 | #' assays[["qvalue"]]$DESeq2 <- p.adjust(cobradata_example@pval$DESeq2, method="BH") 45 | #' groundTruth <- DataFrame( cobradata_example@truth[,c("status", "logFC")] ) 46 | #' colnames(groundTruth) <- names( assays ) 47 | #' colData <- DataFrame( method=colnames(assays[[1]]) ) 48 | #' groundTruth <- groundTruth[rownames(assays[[1]]),] 49 | #' 50 | #' ## constructing a SummarizedBenchmark object 51 | #' sb <- SummarizedBenchmark( 52 | #' assays=assays, colData=colData, 53 | #' groundTruth=groundTruth ) 54 | #' colData(sb)$label <- rownames(colData(sb)) 55 | #' 56 | #' @return A \code{\link{SummarizedBenchmark}} object. 57 | #' @importFrom tidyr gather 58 | #' @importFrom S4Vectors DataFrame elementMetadata 59 | #' @export 60 | #' 61 | SummarizedBenchmark <- function(assays, colData, ftData = NULL, 62 | groundTruth = NULL, performanceMetrics = NULL, 63 | BenchDesign = NULL, ... ) { 64 | if( is( colData, "data.frame" ) ){ 65 | colData <- DataFrame( colData ) 66 | } 67 | if( is( ftData, "data.frame" ) ){ 68 | ftData <- DataFrame( ftData ) 69 | } 70 | if( is( groundTruth, "data.frame" ) ){ 71 | groundTruth <- DataFrame( groundTruth ) 72 | } 73 | if(is.null( groundTruth )){ 74 | groundTruth <- matrix(NA, ncol=length(assays), nrow=nrow(assays[[1]])) 75 | colnames( groundTruth ) <- names(assays) 76 | rownames( groundTruth ) <- rownames( assays[[1]] ) 77 | groundTruth <- DataFrame( groundTruth ) 78 | } 79 | if( length( unique( sapply( assays, nrow ) ) ) > 1 ){ 80 | stop("All metric matrices must have the same number of rows.") 81 | } 82 | elementMetadata(groundTruth) <- DataFrame(colType="groundTruth") 83 | if( is.null( ftData ) ){ 84 | rData <- groundTruth 85 | }else{ 86 | elementMetadata( ftData ) <- DataFrame( colType="featureData" ) 87 | rData <- cbind( groundTruth, ftData ) 88 | } 89 | if( length( unique( sapply( assays, ncol ) ) ) > 1 ){ 90 | stop("All assays must contain the same number of columns") 91 | } 92 | if( is.null(performanceMetrics)){ 93 | performanceMetrics <- vector("list", 2 ) 94 | performanceMetrics <- as( performanceMetrics, "SimpleList" ) 95 | names( performanceMetrics ) <- names(assays) 96 | for( i in seq_along( performanceMetrics ) ){ 97 | performanceMetrics[[i]] <- list() 98 | } 99 | } 100 | elementMetadata( colData ) <- DataFrame( colType="methodInformation" ) 101 | se <- as( SummarizedExperiment( assays, colData=colData, ... ), 102 | "RangedSummarizedExperiment") 103 | rowData(se) <- rData 104 | sb <- new( "SummarizedBenchmark", se, performanceMetrics=performanceMetrics, 105 | BenchDesign = BenchDesign) 106 | sb 107 | } 108 | -------------------------------------------------------------------------------- /R/BDMethod-tidy.R: -------------------------------------------------------------------------------- 1 | #' Tidy BDMethod Data 2 | #' 3 | #' A helper function to extract information for a single 4 | #' or multiple BDMethod object or a list of BDMethod objects. 5 | #' 6 | #' @param obj BDMethod object, list/List of BDMethod objects (e.g. a 7 | #' BDMethodList), or a BenchDesign object 8 | #' @param dat optional data object to use when evaluating any 9 | #' unevaluated expressionsin \code{bdm} meta data. 10 | #' (default = NULL) 11 | #' @param eval logical whether to evaluate any quosures in the meta slot 12 | #' of the BDMethod objects. (default = FALSE) 13 | #' @param label logical whether to add a "label" column to the resulting 14 | #' table containing the names of the methods if \code{obj} was 15 | #' specified as a named list. (default = FALSE) 16 | #' 17 | #' @return 18 | #' A named vector of meta data if only a single BDMethod object specified, else 19 | #' a tibble of meta data for the specified list of methods. 20 | #' 21 | #' @details 22 | #' If any quosures are specified to the "meta" slot of a BDMethod object, 23 | #' the quosure is converted to a text string using \code{rlang::quo_text}. 24 | #' 25 | #' @name tidyBDMethod 26 | #' @import dplyr 27 | #' @importFrom rlang eval_tidy quo_text is_quosure 28 | #' @author Patrick Kimes 29 | NULL 30 | 31 | .tidyBDMethod.bdm <- function(obj, dat, eval) { 32 | if (eval) { 33 | obj@meta <- lapply(obj@meta, function(x) { 34 | tryCatch(rlang::eval_tidy(x, data = dat), 35 | error = function(e) { 36 | message("!! error caught while trying to evaluate BDMethod meta slot !!\n", 37 | "!! original message: \n", 38 | "!! ", conditionMessage(e)) 39 | return(rlang::quo_text(x)) 40 | }) 41 | }) 42 | } else { 43 | quo_meta <- unlist(lapply(obj@meta, rlang::is_quosure)) 44 | if (any(quo_meta)) 45 | obj@meta[quo_meta] <- lapply(obj@meta[quo_meta], rlang::quo_text) 46 | } 47 | 48 | tidyp <- tidyBDMethodParams(obj@params) 49 | tidymf <- tidyBDMethodMeta(obj@meta) 50 | tidym <- tidymf$tidym 51 | 52 | if (is.null(tidymf$tidyf)) { 53 | tidyf <- tidyBDMethodFunction(obj@f) 54 | tidyf$func.pkg.manual <- FALSE 55 | } else { 56 | tidyf <- tidymf$tidyf 57 | tidyf$func.pkg.manual <- TRUE 58 | } 59 | 60 | c(tidyf, tidyp, tidym) 61 | } 62 | 63 | .tidyBDMethod.list <- function(obj, dat, eval, label) { 64 | df <- lapply(obj, tidyBDMethod, dat = dat, eval = eval) 65 | dplyr::bind_rows(df, .id = if(label) { "label" } else { NULL }) 66 | } 67 | 68 | .tidyBDMethod.bd <- function(obj, dat, eval, label) { 69 | if (!is.null(dat)) 70 | tidyBDMethod(obj@methods, dat, eval, label) 71 | else if (!is.null(obj@data) && obj@data@type == "data") 72 | tidyBDMethod(obj@methods, obj@data@data, eval, label) 73 | else 74 | tidyBDMethod(obj@methods, NULL, eval, label) 75 | } 76 | 77 | #' @rdname tidyBDMethod 78 | #' @export 79 | setMethod("tidyBDMethod", signature(obj = "BDMethod"), .tidyBDMethod.bdm) 80 | 81 | #' @rdname tidyBDMethod 82 | #' @export 83 | setMethod("tidyBDMethod", signature(obj = "list"), .tidyBDMethod.list) 84 | 85 | #' @rdname tidyBDMethod 86 | #' @export 87 | setMethod("tidyBDMethod", signature(obj = "SimpleList"), .tidyBDMethod.list) 88 | 89 | #' @rdname tidyBDMethod 90 | #' @export 91 | setMethod("tidyBDMethod", signature(obj = "BenchDesign"), .tidyBDMethod.bd) 92 | 93 | 94 | tidyBDMethodFunction <- function(f) { 95 | fenv <- environment(f) 96 | pkgname <- packageName(fenv) 97 | 98 | if (is.null(pkgname)) { 99 | pkgname <- NA_character_ 100 | pkgvers <- NA_character_ 101 | } else { 102 | pkgvers <- as(packageVersion(pkgname), "character") 103 | } 104 | 105 | list(func.pkg = pkgname, func.pkg.vers = pkgvers) 106 | } 107 | 108 | 109 | tidyBDMethodParams <- function(params) { 110 | if (length(params) == 0) { 111 | return(NULL) 112 | } 113 | 114 | params <- sapply(params, quo_text) 115 | names(params) <- paste0("param.", names(params)) 116 | params 117 | } 118 | 119 | 120 | tidyBDMethodMeta <- function(meta) { 121 | meta_pkgfunc <- meta[["pkg_func"]] 122 | meta_pkgname <- meta[["pkg_name"]] 123 | meta_pkgvers <- meta[["pkg_vers"]] 124 | 125 | if (!is.null(meta_pkgfunc)) { 126 | tidyf <- tidyBDMethodFunction(meta_pkgfunc) 127 | } else if (!is.null(meta_pkgname) | !is.null(meta_pkgvers)) { 128 | tidyf <- list(func.pkg = meta_pkgname, func.pkg.vers = meta_pkgvers) 129 | } else { 130 | tidyf <- NULL 131 | } 132 | 133 | meta <- meta[!(names(meta) %in% c("pkg_func", "pkg_name", "pkg_vers"))] 134 | if (length(meta) > 0) { 135 | names(meta) <- paste0("meta.", names(meta)) 136 | } 137 | 138 | list(tidym = meta, tidyf = tidyf) 139 | } 140 | -------------------------------------------------------------------------------- /_pkgdown.yml: -------------------------------------------------------------------------------- 1 | authors: 2 | Alejandro Reyes: 3 | href: http://alejandroreyes.org 4 | Patrick Kimes: 5 | href: https://www.pkimes.com 6 | 7 | development: 8 | version_label: "info" 9 | version_tooltip: "Bioc Devel" 10 | 11 | navbar: 12 | structure: 13 | left: 14 | - home 15 | - intro 16 | - reference 17 | - vignettes 18 | right: 19 | - news 20 | - search 21 | - github 22 | components: 23 | intro: 24 | text: Quick Start 25 | href: articles/SummarizedBenchmark-Introduction.html 26 | articles: ~ 27 | vignettes: 28 | text: Articles 29 | menu: 30 | - text: General 31 | - text: Introduction 32 | href: articles/SummarizedBenchmark-Introduction.html 33 | - text: Class Details 34 | href: articles/SummarizedBenchmark-ClassDetails.html 35 | - text: Full Case Study 36 | href: articles/SummarizedBenchmark-FullCaseStudy.html 37 | - text: -------- 38 | - text: Features 39 | - text: Error Handling 40 | href: articles/Feature-ErrorHandling.html 41 | - text: Iterative Benchmarking 42 | href: articles/Feature-Iterative.html 43 | - text: Parallelization 44 | href: articles/Feature-Parallel.html 45 | - text: -------- 46 | - text: More Case Studies 47 | - text: scRNA-seq Simulation 48 | href: articles/CaseStudy-SingleCellSimulation.html 49 | - text: RNA-seq Quantification 50 | href: articles/CaseStudy-RNAseqQuantification.html 51 | 52 | toc: 53 | depth: 1 54 | 55 | reference: 56 | - title: Creating Objects 57 | desc: > 58 | Several classes are defined in the package, most importantly, the [BenchDesign](BenchDesign-class.html) and 59 | [SummarizedBenchmark](SummarizedBenchmark-class.html) classes for encapsulating the benchmark design and 60 | benchmark results, repectively. The following set of functions can be used to create new instances of 61 | each class. For more details, see [class definitions](#section-class-definitions) below. 62 | contents: 63 | - '`BenchDesign`' 64 | - '`BDMethod`' 65 | - '`BDMethodList`' 66 | - '`BDData`' 67 | - '`SummarizedBenchmark`' 68 | - title: Executing Benchmarks 69 | desc: > 70 | Benchmark experiments can be executed from either BenchDesign or SummarizedBenchmark objects using the following 71 | set of functions. 72 | contents: 73 | - '`buildBench`' 74 | - '`updateBench`' 75 | - title: BenchDesign Methods 76 | desc: > 77 | Methods and data for benchmark experiments are stored in [BenchDesign](BenchDesign-class.html) objects. 78 | The following functions are available for working with these objects to add, remove, or modify methods 79 | in the design, as well as to compare the methods and data contained in these objects. 80 | contents: 81 | - '`addMethod`' 82 | - '`dropMethod`' 83 | - '`expandMethod`' 84 | - '`modifyMethod`' 85 | - '`printMethod`' 86 | - '`compareBenchDesigns`' 87 | - '`compareBDMethod`' 88 | - '`compareBDData`' 89 | - '`hashBDData`' 90 | - title: SummarizedBenchmark Methods 91 | desc: > 92 | Results from benchmark experiments are stored in [SummarizedBenchmark](SummarizedBenchmark-class.html) objects. 93 | The following functions are available for working with these objects to define benchmark metrics and 94 | evaluate or plot the results of a benchmark experiment. 95 | contents: 96 | - '`groundTruths`' 97 | - '`availableMetrics`' 98 | - '`performanceMetrics`' 99 | - '`addPerformanceMetric`' 100 | - '`estimateMetricsForAssay`' 101 | - '`tidyUpMetrics`' 102 | - '`plotROC`' 103 | - '`plotMethodsOverlap`' 104 | - title: Class Definitions 105 | desc: > 106 | Two core classes, [BenchDesign](BenchDesign-class.html) and 107 | [SummarizedBenchmark](SummarizedBenchmark-class.html) are 108 | defined in the package, along with several smaller classes which help encapsulate data 109 | used in benchmarking ([BDData](BDData-class.html)) and definitions of methods benchmarked 110 | ([BDMethod](BDMethod-class.html), [BDMethodList](BDMethodList-class.html)). 111 | contents: 112 | - '`BenchDesign-class`' 113 | - '`BDData-class`' 114 | - '`BDMethod-class`' 115 | - '`BDMethodList-class`' 116 | - '`SummarizedBenchmark-class`' 117 | - title: Class Setters 118 | desc: > 119 | In addition to using the methods defined above for [working with BenchDesign objects](#section-benchdesign-methods) 120 | and [working with SummarizedBenchmark](#section-summarizedbenchmark-methods), some slots and values of 121 | the classes can be directly modified using the following functions. 122 | contents: 123 | - '`BDData<-`' 124 | - '`BDMethod<-`' 125 | - '`BDMethodList<-`' 126 | - '`performanceMetrics<-`' 127 | - '`groundTruths<-`' 128 | - '`mcols<-,SummarizedBenchmark-method`' 129 | - '`assayNames<-,SummarizedBenchmark,character-method`' 130 | - title: Data Sets 131 | desc: > 132 | Example data sets are included to illustrate the functionality of the package in the 133 | various vignettes and case studies. 134 | contents: 135 | - '`allSB`' 136 | - '`sb`' 137 | - '`tdat`' 138 | -------------------------------------------------------------------------------- /R/SummarizedBenckmark-accessors.R: -------------------------------------------------------------------------------- 1 | performanceMetricsSB <- function( object, assay=NULL ){ 2 | validObject( object ) 3 | if( is.null( assay ) ){ 4 | return( object@performanceMetrics ) 5 | }else{ 6 | inAssay <- assay %in% assayNames( object ) 7 | if( !all( inAssay ) ){ 8 | stop( sprintf( "Performance metric(s) '%s' not found.", assay[!inAssay] ) ) 9 | } 10 | return( object@performanceMetrics[assay] ) 11 | } 12 | } 13 | 14 | #' @rdname performanceMetrics 15 | #' @exportMethod "performanceMetrics" 16 | setMethod("performanceMetrics", 17 | signature(object = "SummarizedBenchmark"), performanceMetricsSB) 18 | 19 | #' @rdname performanceMetrics-setter 20 | #' @exportMethod "performanceMetrics<-" 21 | setReplaceMethod("performanceMetrics", 22 | signature(object="SummarizedBenchmark", value="SimpleList"), 23 | function( object, value ) { 24 | object@performanceMetrics <- value 25 | validObject(object) 26 | object 27 | }) 28 | 29 | #' Set assay names in SummarizedBenchmark object 30 | #' 31 | #' Modifies the assay names of a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 32 | #' object. 33 | #' 34 | #' @param x A \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. 35 | #' @param value A character vector. 36 | #' @param ... Futher arguments, perhaps used by methods. 37 | #' 38 | #' @return 39 | #' Modfied \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. 40 | #' 41 | #' @examples 42 | #' data(sb) 43 | #' assayNames(sb)[2] <- "log2FC" 44 | #' 45 | #' @seealso \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 46 | #' @rdname assayNames-setter 47 | #' @importMethodsFrom SummarizedExperiment assayNames "assayNames<-" 48 | #' @exportMethod "assayNames<-" 49 | #' @author Alejandro Reyes 50 | setReplaceMethod( "assayNames", c("SummarizedBenchmark", "character"), 51 | function(x, ..., value) 52 | { 53 | oldNames <- names( assays( x, withDimnames=FALSE ) ) 54 | names( assays( x, withDimnames=FALSE ) ) <- value 55 | newNames <- names( assays( x, withDimnames=FALSE ) ) 56 | mm <- match( names( x@performanceMetrics ), oldNames ) 57 | names( x@performanceMetrics )[mm] <- newNames 58 | truthCol <- elementMetadata( rowData( x ) )$colType == "groundTruth" 59 | truthCol[is.na(truthCol)] <- FALSE 60 | if( any( truthCol ) ){ 61 | mm <- match( colnames( rowData( x ) )[truthCol], oldNames ) 62 | colnames(rowData( x ))[truthCol][mm] <- newNames 63 | } 64 | x 65 | } ) 66 | 67 | 68 | #' Set meta data columns in SummarizedBenchmark object 69 | #' 70 | #' Modifies the \code{mcols} slot of a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 71 | #' object. 72 | #' 73 | #' @param x A \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. 74 | #' @param value A DataFrame of meta data. 75 | #' @param ... Futher arguments, perhaps used by methods. 76 | #' 77 | #' @return 78 | #' Modfied \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. 79 | #' 80 | #' @seealso \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 81 | #' @rdname mcols-setter 82 | #' @importMethodsFrom S4Vectors mcols elementMetadata "mcols<-" "elementMetadata<-" 83 | #' @exportMethod "mcols<-" 84 | #' @author Alejandro Reyes 85 | setReplaceMethod("mcols", "SummarizedBenchmark", 86 | function(x, ..., value) 87 | { 88 | x <- BiocGenerics:::replaceSlots(x, 89 | rowRanges=local({ 90 | r <- rowRanges( x ) 91 | if( length( value ) > 0 ){ 92 | wc <- colnames( value ) %in% assayNames( x ) 93 | mcols( value )$colType <- ifelse( wc, "groundTruth", "featureData" ) 94 | } 95 | mcols( r ) <- value 96 | r 97 | }), 98 | check=FALSE) 99 | x 100 | }) 101 | 102 | #' @rdname groundTruths 103 | #' @exportMethod "groundTruths" 104 | setMethod( 105 | "groundTruths", 106 | "SummarizedBenchmark", 107 | function( object, ... ){ 108 | stopifnot( is( object, "SummarizedBenchmark") ) 109 | cols <- mcols( mcols(object) )$colType == "groundTruth" 110 | mcols(object, ...)[,cols, drop=FALSE] 111 | } 112 | ) 113 | 114 | #' @rdname groundTruths-setter 115 | #' @exportMethod "groundTruths<-" 116 | setReplaceMethod( 117 | "groundTruths", 118 | "SummarizedBenchmark", 119 | function(object, ..., value){ 120 | mcols(object, ...) <- value 121 | object 122 | } 123 | ) 124 | 125 | #' @rdname BDMethodList 126 | #' @exportMethod "BDMethodList" 127 | setMethod("BDMethodList", 128 | signature(x = "SummarizedBenchmark"), 129 | function(..., x) { 130 | BenchDesign(x)@methods 131 | }) 132 | 133 | 134 | #' @rdname BenchDesign 135 | #' @exportMethod "BenchDesign" 136 | setMethod("BenchDesign", signature(methods = "SummarizedBenchmark", data = "ANY"), 137 | function(methods, data) { 138 | if (!is.null(data)) 139 | message("Note: Ignoring specified data and using SummarizedBenchmark object.") 140 | bd <- methods@BenchDesign 141 | if (is.null(bd)) 142 | bd <- BenchDesign() 143 | bd 144 | }) 145 | -------------------------------------------------------------------------------- /R/addMethod.R: -------------------------------------------------------------------------------- 1 | #' Add method to BenchDesign object 2 | #' 3 | #' @description 4 | #' Takes a \code{\link[=BenchDesign-class]{BenchDesign}} object 5 | #' and the definition of a new method for benchmarking and returns 6 | #' the original \code{\link[=BenchDesign-class]{BenchDesign}} with 7 | #' the new method included. 8 | #' 9 | #' At a minimum, a method label (`label =`), and the 10 | #' workhorse function for the method (`func =`) must be specified 11 | #' for the new method. 12 | #' 13 | #' Parameters for the method must be specified as a 14 | #' \code{\link[rlang:quotation]{quos}} named list of `parameter = value` pairs 15 | #' mapping entries in the benchmarking data to the function parameters. 16 | #' For users familiar with the \pkg{ggplot2} package, this can be 17 | #' viewed similar to the \code{aes =} mapping of data to geometry 18 | #' parameters. 19 | #' 20 | #' An optional secondary function, `post`, can be specified if 21 | #' the output of the workhorse function, `func`, needs to be 22 | #' further processed. As an example, `post` may be a simple 23 | #' "getter" function for accessing the column of interest from 24 | #' the large object returned by `func`. 25 | #' 26 | #' @param bd \code{\link[=BenchDesign-class]{BenchDesign}} object. 27 | #' @param label Character name for the method. 28 | #' @param func Primary function to be benchmarked. 29 | #' @param params Named quosure list created using \code{\link[rlang:quotation]{quos}} of 30 | #' `parameter = value` pairs to be passed to `func`. 31 | #' @param post Optional post-processing function that takes 32 | #' results of `func` as input. Ignored if \code{NULL}. 33 | #' If multiple assays (metrics) should be generated for each 34 | #' method, this can be accomplished by specifying a named 35 | #' list of post-processing functions, one for each assay. 36 | #' (default = \code{NULL}) 37 | #' @param meta Optional metadata information for method to be 38 | #' included in `colData` of \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object 39 | #' generated using \code{link{buildBench}}. See Details for more 40 | #' information. Ignored if \code{NULL}. (default = \code{NULL}) 41 | #' 42 | #' @return 43 | #' Modified \code{\link[=BenchDesign-class]{BenchDesign}} object with 44 | #' new method added. 45 | #' 46 | #' @details 47 | #' The optional `meta` parameter accepts a named list of metadata 48 | #' tags to be included for the method in the resulting 49 | #' \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 50 | #' object. This can be useful for two primary cases. First, it can help keep 51 | #' analyses better organized by allowing the specification of additional 52 | #' information that should be stored with methods, e.g. a tag for "method type" 53 | #' or descriptive information on why the method was included in the comparison. 54 | #' Second, and more improtantly, the `meta` parameter can be used to overwrite 55 | #' the package and version information that is automatically extracted from the 56 | #' function specified to `func`. This is particularly useful when the function 57 | #' passed to `func` is a wrapper for a script in (or outside of) R, and the 58 | #' appropriate package and version information can't be directly pulled from 59 | #' `func`. In this case, the user can either manually specify the `"pkg_name"` 60 | #' and `"pkg_vers"` values to `meta` as a list, or specify a separate function 61 | #' that should be used to determine the package name and version. If a separate 62 | #' function should be used, it should be passed to `meta` as a list entry 63 | #' with the name `pkg_func` and first quoted using \code{\link[rlang:quotation]{quo}}, e.g. 64 | #' `list(pkg_func = quo(p.adjust))`. 65 | #' 66 | #' @examples 67 | #' ## create example data set of p-values 68 | #' df <- data.frame(pval = runif(100)) 69 | #' 70 | #' ## example calculating qvalue from pvalues 71 | #' 72 | #' ## using standard call 73 | #' qv <- qvalue::qvalue(p = df$pval) 74 | #' qv <- qv$qvalue 75 | #' 76 | #' ## adding same method to BenchDesign 77 | #' bench <- BenchDesign(data = df) 78 | #' bench <- addMethod(bench, 79 | #' label = "qv", 80 | #' func = qvalue::qvalue, 81 | #' post = function(x) { x$qvalue }, 82 | #' params = rlang::quos(p = pval)) 83 | #' 84 | #' @seealso \code{\link{modifyMethod}}, \code{\link{expandMethod}}, \code{\link{dropMethod}} 85 | #' @md 86 | #' @importFrom rlang is_quosures quos enquo 87 | #' @export 88 | #' @author Patrick Kimes 89 | addMethod <- function(bd, label, func, params = rlang::quos(), 90 | post = NULL, meta = NULL) { 91 | UseMethod("addMethod") 92 | } 93 | 94 | #' @export 95 | addMethod.BenchDesign <- function(bd, label, func, params = rlang::quos(), 96 | post = NULL, meta = NULL) { 97 | if (!rlang::is_quosures(params)) { 98 | stop("Please supply 'func' parameters to 'params =' as ", 99 | "a list of quosures using rlang::quos.\n", 100 | "e.g. params = quos(param1 = x, param2 = y)") 101 | } 102 | ## capture input 103 | qf <- rlang::enquo(func) 104 | 105 | ## add to bench 106 | bd@methods[[label]] <- BDMethod(x = qf, params = params, post = post, meta = meta) 107 | bd 108 | } 109 | -------------------------------------------------------------------------------- /docs/pkgdown.css: -------------------------------------------------------------------------------- 1 | /* Sticky footer */ 2 | 3 | /** 4 | * Basic idea: https://philipwalton.github.io/solved-by-flexbox/demos/sticky-footer/ 5 | * Details: https://github.com/philipwalton/solved-by-flexbox/blob/master/assets/css/components/site.css 6 | * 7 | * .Site -> body > .container 8 | * .Site-content -> body > .container .row 9 | * .footer -> footer 10 | * 11 | * Key idea seems to be to ensure that .container and __all its parents__ 12 | * have height set to 100% 13 | * 14 | */ 15 | 16 | html, body { 17 | height: 100%; 18 | } 19 | 20 | body > .container { 21 | display: flex; 22 | height: 100%; 23 | flex-direction: column; 24 | 25 | padding-top: 60px; 26 | } 27 | 28 | body > .container .row { 29 | flex: 1 0 auto; 30 | } 31 | 32 | footer { 33 | margin-top: 45px; 34 | padding: 35px 0 36px; 35 | border-top: 1px solid #e5e5e5; 36 | color: #666; 37 | display: flex; 38 | flex-shrink: 0; 39 | } 40 | footer p { 41 | margin-bottom: 0; 42 | } 43 | footer div { 44 | flex: 1; 45 | } 46 | footer .pkgdown { 47 | text-align: right; 48 | } 49 | footer p { 50 | margin-bottom: 0; 51 | } 52 | 53 | img.icon { 54 | float: right; 55 | } 56 | 57 | img { 58 | max-width: 100%; 59 | } 60 | 61 | /* Fix bug in bootstrap (only seen in firefox) */ 62 | summary { 63 | display: list-item; 64 | } 65 | 66 | /* Typographic tweaking ---------------------------------*/ 67 | 68 | .contents .page-header { 69 | margin-top: calc(-60px + 1em); 70 | } 71 | 72 | /* Section anchors ---------------------------------*/ 73 | 74 | a.anchor { 75 | margin-left: -30px; 76 | display:inline-block; 77 | width: 30px; 78 | height: 30px; 79 | visibility: hidden; 80 | 81 | background-image: url(./link.svg); 82 | background-repeat: no-repeat; 83 | background-size: 20px 20px; 84 | background-position: center center; 85 | } 86 | 87 | .hasAnchor:hover a.anchor { 88 | visibility: visible; 89 | } 90 | 91 | @media (max-width: 767px) { 92 | .hasAnchor:hover a.anchor { 93 | visibility: hidden; 94 | } 95 | } 96 | 97 | 98 | /* Fixes for fixed navbar --------------------------*/ 99 | 100 | .contents h1, .contents h2, .contents h3, .contents h4 { 101 | padding-top: 60px; 102 | margin-top: -40px; 103 | } 104 | 105 | /* Static header placement on mobile devices */ 106 | @media (max-width: 767px) { 107 | .navbar-fixed-top { 108 | position: absolute; 109 | } 110 | .navbar { 111 | padding: 0; 112 | } 113 | } 114 | 115 | 116 | /* Sidebar --------------------------*/ 117 | 118 | #sidebar { 119 | margin-top: 30px; 120 | } 121 | #sidebar h2 { 122 | font-size: 1.5em; 123 | margin-top: 1em; 124 | } 125 | 126 | #sidebar h2:first-child { 127 | margin-top: 0; 128 | } 129 | 130 | #sidebar .list-unstyled li { 131 | margin-bottom: 0.5em; 132 | } 133 | 134 | .orcid { 135 | height: 16px; 136 | vertical-align: middle; 137 | } 138 | 139 | /* Reference index & topics ----------------------------------------------- */ 140 | 141 | .ref-index th {font-weight: normal;} 142 | 143 | .ref-index td {vertical-align: top;} 144 | .ref-index .icon {width: 40px;} 145 | .ref-index .alias {width: 40%;} 146 | .ref-index-icons .alias {width: calc(40% - 40px);} 147 | .ref-index .title {width: 60%;} 148 | 149 | .ref-arguments th {text-align: right; padding-right: 10px;} 150 | .ref-arguments th, .ref-arguments td {vertical-align: top;} 151 | .ref-arguments .name {width: 20%;} 152 | .ref-arguments .desc {width: 80%;} 153 | 154 | /* Nice scrolling for wide elements --------------------------------------- */ 155 | 156 | table { 157 | display: block; 158 | overflow: auto; 159 | } 160 | 161 | /* Syntax highlighting ---------------------------------------------------- */ 162 | 163 | pre { 164 | word-wrap: normal; 165 | word-break: normal; 166 | border: 1px solid #eee; 167 | } 168 | 169 | pre, code { 170 | background-color: #f8f8f8; 171 | color: #333; 172 | } 173 | 174 | pre code { 175 | overflow: auto; 176 | word-wrap: normal; 177 | white-space: pre; 178 | } 179 | 180 | pre .img { 181 | margin: 5px 0; 182 | } 183 | 184 | pre .img img { 185 | background-color: #fff; 186 | display: block; 187 | height: auto; 188 | } 189 | 190 | code a, pre a { 191 | color: #375f84; 192 | } 193 | 194 | a.sourceLine:hover { 195 | text-decoration: none; 196 | } 197 | 198 | .fl {color: #1514b5;} 199 | .fu {color: #000000;} /* function */ 200 | .ch,.st {color: #036a07;} /* string */ 201 | .kw {color: #264D66;} /* keyword */ 202 | .co {color: #888888;} /* comment */ 203 | 204 | .message { color: black; font-weight: bolder;} 205 | .error { color: orange; font-weight: bolder;} 206 | .warning { color: #6A0366; font-weight: bolder;} 207 | 208 | /* Clipboard --------------------------*/ 209 | 210 | .hasCopyButton { 211 | position: relative; 212 | } 213 | 214 | .btn-copy-ex { 215 | position: absolute; 216 | right: 0; 217 | top: 0; 218 | visibility: hidden; 219 | } 220 | 221 | .hasCopyButton:hover button.btn-copy-ex { 222 | visibility: visible; 223 | } 224 | 225 | /* mark.js ----------------------------*/ 226 | 227 | mark { 228 | background-color: rgba(255, 255, 51, 0.5); 229 | border-bottom: 2px solid rgba(255, 153, 51, 0.3); 230 | padding: 1px; 231 | } 232 | 233 | /* vertical spacing after htmlwidgets */ 234 | .html-widget { 235 | margin-bottom: 10px; 236 | } 237 | -------------------------------------------------------------------------------- /R/printUpdateBench.R: -------------------------------------------------------------------------------- 1 | ## Print Comparison of BenchDesign objects 2 | ## 3 | ## Internal function for printing the results of a comparison of the 4 | ## methods executed in a SummarizedBenchmark object and a new BenchDesign 5 | ## object. 6 | ## 7 | ## @param sb a SummarizedBenchmark object 8 | ## @param bd a BenchDesign object 9 | ## @param version logical whether to re-run methods with only package 10 | ## version differences. (default = FALSE) 11 | ## @param keepAll logical whether to keep methods run in original SummarizedBenchmark 12 | ## but not in new BenchDesign. Only used if \code{bd} is not NULL. (default = TRUE) 13 | ## 14 | ## logical value indicating whether the two objects have 15 | ## methods producing the same meta data and data with the 16 | ## same MD5 hashes. 17 | ## 18 | ## @author Patrick Kimes 19 | .printUpdateBench <- function(sb, bd, version, keepAll) { 20 | stopifnot(is(sb, "SummarizedBenchmark")) 21 | stopifnot(is(bd, "BenchDesign")) 22 | 23 | res <- compareBenchDesigns(sb, bd) 24 | 25 | metres <- res$methods$res 26 | metres <- dplyr::mutate(metres, rerun = !f | !meta | !params | !post) 27 | if (version) { 28 | metres <- dplyr::mutate(metres, rerun = rerun | !version) 29 | } 30 | metres <- dplyr::mutate(metres, rerun = rerun | overlap == "yOnly") 31 | metres <- dplyr::mutate(metres, oldfn = overlap != "yOnly") 32 | 33 | ## need to rerun all if data is new 34 | if (!isTRUE(res$data$data)) { 35 | metres <- dplyr::mutate(metres, rerun = TRUE) 36 | } 37 | 38 | sbdat <- BDData(sb) 39 | bddat <- BDData(bd) 40 | 41 | cat(stringr::str_pad(crayon::bold("Update SummarizedBenchmark (dryrun) "), 70, "right", "-"), "\n") 42 | if (isTRUE(res$data$data)) { 43 | if (!isFALSE(res$data$type) && res$data$type == "md5hash") { 44 | cat(crayon::yellow$bold(" benchmark data:"), crayon::yellow("unchanged (full data missing)\n")) 45 | cat(" MD5 hash: ") 46 | cat(sbdat@data, "\n") 47 | cat(" names: ??\n") 48 | } else { 49 | cat(crayon::green$bold(" benchmark data:"), crayon::green("unchanged\n")) 50 | cat(" MD5 hash: ") 51 | cat(hashBDData(sbdat)@data, "\n") 52 | if (sbdat@type == "data") 53 | cat(" names:", names(sbdat@data), "\n") 54 | else 55 | cat(" names:", names(bddat@data), "\n") 56 | } 57 | } else { 58 | cat(crayon::red$bold(" benchmark data:"), crayon::red("changed ")) 59 | if (bddat@type == "md5hash") { 60 | cat(crayon::red("(full data missing - can't run)\n")) 61 | cat(" BenchDesign\n") 62 | cat(" MD5 hash: ", bddat@data, "\n") 63 | cat(" names: ??\n") 64 | } else { 65 | cat("\n") 66 | cat(" BenchDesign\n") 67 | cat(" MD5 hash: ", hashBDData(bddat)@data, "\n") 68 | cat(" names:", names(bddat@data), "\n") 69 | } 70 | cat(" SummarizedBenchmark\n") 71 | cat(" MD5 hash: ", hashBDData(sbdat)@data, "\n") 72 | if (sbdat@type == "md5hash") 73 | cat(" names: ??\n") 74 | else 75 | cat(" names:", names(sbdat@data), "\n") 76 | } 77 | 78 | metres <- dplyr::mutate_if(metres, is.logical, `!`) 79 | metres <- dplyr::mutate(metres, rerun = !rerun) 80 | metres <- dplyr::mutate_if(metres, is.logical, ~ ifelse(., "Y", "N")) 81 | if (!keepAll) 82 | metres$rerun[metres$overlap == "xOnly"] <- "Drop" 83 | else 84 | metres$rerun[metres$overlap == "xOnly"] <- "N" 85 | 86 | cat(crayon::bold(" benchmark methods:\n")) 87 | if (nrow(metres)) { 88 | header1 <- paste0(stringr::str_pad(" |", 18, "right"), stringr::str_pad("| Need to", 10), " |", 89 | stringr::str_pad("Outdated", 31, "both"), "|\n") 90 | header2 <- .methodrow(c("Method", "(Re)Run", "Func", "Param", "Meta", "Post", "Vers")) 91 | cat(crayon::bold(header1)) 92 | cat(crayon::bold(header2)) 93 | for (i in seq_len(nrow(metres))) { 94 | if (metres$overlap[i] == "Both") { 95 | istr <- unlist(metres[i, c("label", "rerun", "f", "params", "meta", "post", "version")]) 96 | istr <- .methodrow(istr) 97 | cat(istr) 98 | } else { 99 | istr <- c(unlist(metres[i, c("label", "rerun")]), rep("-", 5)) 100 | istr <- .methodrow(istr) 101 | cat(istr) 102 | } 103 | } 104 | } else { 105 | cat(" none\n") 106 | } 107 | invisible(res) 108 | } 109 | 110 | ## standard row format 111 | .methodrow <- function(x, n) { 112 | method_label <- stringr::str_pad(stringr::str_trunc(x[1], 12), 12, "right") 113 | if (x[2] == "Y") { 114 | method_label <- crayon::red(method_label) 115 | } else if (x[2] == "N") { 116 | method_label <- crayon::green(method_label) 117 | } 118 | paste0(" | ", method_label, " |", .loglab(x[2], 8), " |", 119 | .loglab(x[3], 6), .loglab(x[4], 6), .loglab(x[5], 6), .loglab(x[6], 6), 120 | .loglab(x[7], 6), " |\n") 121 | } 122 | 123 | ## standard column format 124 | .loglab <- function(x, n) { 125 | xp <- stringr::str_pad(x, n, "left") 126 | if (x == "Y") 127 | crayon::red(xp) 128 | else if (x == "N") 129 | crayon::green(xp) 130 | else 131 | xp 132 | } 133 | 134 | -------------------------------------------------------------------------------- /vignettes/Feature-ErrorHandling.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Feature: Error Handling" 3 | author: "Patrick K. Kimes, Alejandro Reyes" 4 | date: "`r BiocStyle::doc_date()`" 5 | package: "`r BiocStyle::pkg_ver('SummarizedBenchmark')`" 6 | abstract: > 7 | "Method execution in the `SummarizedBenchmark` framework is handled by the `buildBench()` function. Occassionally, errors occur when running several methods on a new data set. The approach to error handling implemented in the `buildBench()` function is described in this vignette along with options to disable error handling. SummarizedBenchmark package version: `r packageVersion("SummarizedBenchmark")`" 8 | output: 9 | BiocStyle::html_document: 10 | highlight: pygments 11 | toc: true 12 | fig_width: 5 13 | bibliography: library.bib 14 | vignette: > 15 | %\VignetteIndexEntry{Feature: Error Handling} 16 | %\VignetteEncoding{UTF-8} 17 | %\VignetteEngine{knitr::rmarkdown} 18 | editor_options: 19 | chunk_output_type: inline 20 | --- 21 | 22 | ```{r echo = FALSE, include = FALSE} 23 | knitr::opts_chunk$set(tidy = FALSE, cache = TRUE, dev = "png", 24 | message = FALSE, error = FALSE, warning = TRUE) 25 | ``` 26 | 27 | # Introduction 28 | 29 | When running a large benchmark study, not uncommonly, a single or a small subset of methods may fail during execution. This may be the result of misspecified parameters, an underlying bug in the software, or any number of other reasons. By default, errors thrown by methods which fail during `buildBench()` or `updateBench()` (see **Feature: Iterative Benchmarking** for details on `updateBench()`) are caught and handled in a user-friendly way. As long as a single method executes without any errors, a _SummarizedBenchmark_ object is returned as usual, with the assay columns of failed methods set to `NA`. Additionally, the corresponding error messages are stored in the metadata of the object for reference. 30 | 31 | # Simple Case Study 32 | 33 | ```{r} 34 | library(SummarizedBenchmark) 35 | library(magrittr) 36 | ``` 37 | 38 | As an example, consider the following example where we run case where we benchmark two simple methods. The first, `slowMethod` draws 5 random normal samples after waiting 5 seconds, and the second, `fastMethod` draws 5 random normal samples immediately. Each method is then passed through two post-processing functions, `keepSlow` and `makeSlower`, and `keepFast` and `makeSlower`, respectively. This results in three partially overlapping assays, `keepSlow`, `keepFast` and `makeSlower`. With this example, we also demonstrate how mismatched assays are handled across methods. 39 | 40 | ```{r} 41 | bdslow <- BenchDesign(data = tdat) %>% 42 | addMethod("slowMethod", function() { Sys.sleep(5); rnorm(5) }, 43 | post = list(keepSlow = identity, 44 | makeSlower = function(x) { Sys.sleep(5); x })) %>% 45 | addMethod("fastMethod", function() { rnorm(5) }, 46 | post = list(keepFast = identity, 47 | makeSlower = function(x) { Sys.sleep(5); x })) 48 | ``` 49 | 50 | We run these methods in parallel using `parallel = TRUE` and specify a timeout limit of 1 second for the `BPPARAM`. Naturally, `slowMethod` will fail, and `fastMethod` will fail during the `makeSlower` post-processing function. 51 | 52 | ```{r, message = TRUE} 53 | bpp <- SerialParam() 54 | bptimeout(bpp) <- 1 55 | 56 | sbep <- buildBench(bdslow, parallel = TRUE, BPPARAM = bpp) 57 | ``` 58 | 59 | Notice that during the execution process, errors caught by `buildBench()` are printed to the console along with the name of the failed method and post-processing function when appropriate. 60 | 61 | We can verify that a valid _SummarizedBenchmark_ object is still returned with the the remaining results. 62 | 63 | ```{r} 64 | sbep 65 | ``` 66 | 67 | We can also check the values of the assays. 68 | 69 | ```{r} 70 | sapply(assayNames(sbep), assay, x = sbep, simplify = FALSE) 71 | ``` 72 | 73 | Notice that most columns contain only `NA` values. These columns correspond to both methods which returned errors, as well as methods missing post-processing functions, e.g. no `keepSlow` function was defined for the `fastMethod` method. While the `NA` values cannot be used to distinguish the sources of the `NA` values, this is documented in the `sessions` list of the _SummarizedBenchmark_ metadata. While the `sessions` object is a list containing information for all previous sessions, we are only interested in the current, first session. (For more details on why multiple sessions may be run, see the **Feature: Iterative Benchmarking** vignette.) 74 | 75 | ```{r} 76 | names(metadata(sbep)$sessions[[1]]) 77 | ``` 78 | 79 | In `sessions`, there is a `"results"` entry which includes a summary of the results for each combination of method and post-processing function (assay). The entries of `results` can take one of three values: `"success"`, `"missing"`, or an error message of class `buildbench-error`. The easiest way to view these resultsis by passing the `results` to the base R function, `simplify2array()`. 80 | 81 | ```{r} 82 | sess1res <- metadata(sbep)$sessions[[1]]$results 83 | simplify2array(sess1res) 84 | ``` 85 | 86 | In the returned table, columns correspond to methods, and rows correspond to assays. We clearly see that many of the methods failed due to exceeding the specified time limit. If we check one of these entries more closesly, we see that it is indeed a `buildbench-error` object that occurred (`"origin"`) during the `"main"` function. 87 | 88 | ```{r} 89 | sess1res$slowMethod$keepSlow 90 | ``` 91 | 92 | # Disable Handling 93 | 94 | If this error handling is not wanted, and the user would like the benchmark experiment to terminate when an error is thrown, then optional parameter `catchErrors = FALSE` can be specified to eiher `buildBench()` or `updateBench()`. Generally, this is advised against as the outputs computed for all non-failing methods will also be lost. As a result, the entire benchmarking experiment will need to be re-executed. 95 | 96 | # References 97 | 98 | -------------------------------------------------------------------------------- /R/expandMethod.R: -------------------------------------------------------------------------------- 1 | #' Expand method in BenchDesign object 2 | #' 3 | #' @description 4 | #' Takes a \code{\link[=BenchDesign-class]{BenchDesign}} object, the name of an 5 | #' existing method, and new parameter specifications, 6 | #' and returns a modified \code{\link[=BenchDesign-class]{BenchDesign}} 7 | #' object with new methods added. The named method is "expanded" to mutliple methods 8 | #' according to the specified set of parameters. 9 | #' 10 | #' @param bd \code{\link[=BenchDesign-class]{BenchDesign}} object. 11 | #' @param label Character name of method to be expanded. 12 | #' @param params Named list of quosure lists specifying the label of the 13 | #' new methods to be added to the \code{\link[=BenchDesign-class]{BenchDesign}}, 14 | #' and the set of 15 | #' parameters to overwrite in the original method definition for 16 | #' each new method. Alternatively, if `onlyone` is non-\code{NULL}, a single quosure 17 | #' list with `name = value` pairs specifying the label of the new methods and 18 | #' the values to use for overwriting the parameter specified in `onlyone`. 19 | #' @param onlyone Character name of a parameter to be modified. Only specify 20 | #' if just a single parameter should be replaced in the original 21 | #' method definition. Ignored if \code{NULL}. (default = \code{NULL}) 22 | #' @param .replace Logical whether original `label` method should be removed. 23 | #' (default = \code{FALSE}) 24 | #' @param .overwrite Logical whether to overwrite the existing list of 25 | #' parameters (`TRUE`) or to simply add the new parameters to the existing 26 | #' list (`FALSE`). (default = `FALSE`) 27 | #' 28 | #' @return 29 | #' Modified \code{\link[=BenchDesign-class]{BenchDesign}} object with new methods with 30 | #' specified parameters added. 31 | #' 32 | #' @examples 33 | #' ## empty BenchDesign 34 | #' bench <- BenchDesign() 35 | #' 36 | #' ## add basic 'padjust' method 37 | #' bench <- addMethod(bench, label = "padjust", 38 | #' func = p.adjust, 39 | #' params = rlang::quos(p = pval, method = "none")) 40 | #' 41 | #' ## modify multiple parameters - params is a list of quosure lists 42 | #' newparams <- list(bonf = rlang::quos(p = round(pval, 5), method = "bonferonni"), 43 | #' bh = rlang::quos(p = round(pval, 3), method = "BH")) 44 | #' bench_exp <- expandMethod(bench, label = "padjust", params = newparams) 45 | #' BDMethodList(bench_exp) 46 | #' 47 | #' ## only modify a single parameter - params is a quosure list 48 | #' newparams <- rlang::quos(bonf = "bonferonni", BH = "BH") 49 | #' bench_exp <- expandMethod(bench, label = "padjust", onlyone = "method", params = newparams) 50 | #' BDMethodList(bench_exp) 51 | #' 52 | #' @seealso \code{\link{modifyMethod}}, \code{\link{addMethod}}, \code{\link{dropMethod}} 53 | #' @md 54 | #' @importFrom rlang is_quosures quos := 55 | #' @export 56 | #' @author Patrick Kimes 57 | expandMethod <- function(bd, label, params, onlyone = NULL, 58 | .replace = FALSE, .overwrite = FALSE) { 59 | UseMethod("expandMethod") 60 | } 61 | 62 | #' @export 63 | expandMethod.BenchDesign <- function(bd, label, params, onlyone = NULL, 64 | .replace = FALSE, .overwrite = FALSE) { 65 | ## parse new inputs 66 | if (rlang::is_quosures(params)) { 67 | if (is.null(onlyone)) { 68 | stop("If onlyone = NULL, a named list of parameter lists must be supplied to ", 69 | "'params =' as a list of quosure lists created using rlang::quos.\n", 70 | "e.g. params = list(new_method1 = quos(..), new_method2 = quos(..))") 71 | } 72 | qd <- lapply(1:length(params), function(zi) { 73 | rlang::quos(!! onlyone := !! params[[zi]]) 74 | }) 75 | names(qd) <- names(params) 76 | } else if (is.list(params) & all(sapply(params, rlang::is_quosures))) { 77 | if (!is.null(onlyone)) { 78 | stop("If onlyone is non-NULL, a list of parameter values must be supplied to ", 79 | "'params =' as a list of quosures created using rlang::quos.\n", 80 | "e.g. params = quos(new_method1 = new_value1, new_method2 = new_value2)") 81 | } 82 | qd <- params 83 | } else { 84 | if (is.null(onlyone)) { 85 | stop("If 'onlyone = NULL', a named list of parameter lists must be supplied to ", 86 | "'params =' as a list of quosure lists created using rlang::quos.\n", 87 | "e.g. params = list(new_method1 = quos(..), new_method2 = quos(..))") 88 | } else { 89 | stop("If 'onlyone' is non-NULL, a list of parameter values must be supplied to ", 90 | "'params =' as a list of quosures created using rlang::quos.\n", 91 | "e.g. params = quos(new_method1 = new_value1, new_method2 = new_value2)") 92 | } 93 | } 94 | 95 | ## verify that parameter names are valid 96 | if (is.null(names(qd)) | any(nchar(names(qd)) == 0)) { 97 | stop("New parameter values must be named.") 98 | } 99 | if (any(names(qd) %in% names(bd@methods))) { 100 | stop("New method names should not overlap with names of current methods.") 101 | } 102 | if (any(duplicated(names(qd)))) { 103 | stop("New method names must be unique.") 104 | } 105 | 106 | ## verify that method definition already exists 107 | if(!(label %in% names(bd@methods))) { 108 | stop("Specified method is not defined in BenchDesign.") 109 | } 110 | bm <- bd@methods[[label]] 111 | 112 | ## handle all using same named list format 113 | zl <- lapply(qd, .modmethod, m = bm, .overwrite = .overwrite) 114 | names(zl) <- names(qd) 115 | 116 | ## drop source method 117 | if (.replace) { 118 | bd <- dropMethod(bd, label) 119 | } 120 | 121 | bd@methods <- c(bd@methods, zl) 122 | bd 123 | } 124 | -------------------------------------------------------------------------------- /man/buildBench.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/buildBench.R 3 | \name{buildBench} 4 | \alias{buildBench} 5 | \title{Execute BenchDesign} 6 | \usage{ 7 | buildBench(bd, data = NULL, truthCols = NULL, ftCols = NULL, 8 | sortIDs = FALSE, keepData = FALSE, catchErrors = TRUE, 9 | parallel = FALSE, BPPARAM = bpparam()) 10 | } 11 | \arguments{ 12 | \item{bd}{\code{\link[=BenchDesign-class]{BenchDesign}} object.} 13 | 14 | \item{data}{Data set to be used for benchmarking, will take priority over 15 | data set specified to \code{\link[=BenchDesign-class]{BenchDesign}} object. 16 | Ignored if \code{NULL}. (default = \code{NULL})} 17 | 18 | \item{truthCols}{Character vector of column names in data set corresponding to 19 | ground truth values for each assay. If specified, column will be added to 20 | the \code{groundTruth} DataFrame of the returned 21 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. 22 | If the \code{\link[=BenchDesign-class]{BenchDesign}} includes only a single assay, the same name 23 | will be used for the assay. If the \code{\link[=BenchDesign-class]{BenchDesign}} 24 | includes multiple assays, 25 | to map data set columns with assays, the vector must have names corresponding 26 | to the assay names specified to the \code{post} parameter at each 27 | \code{addMethod} call. (default = \code{NULL})} 28 | 29 | \item{ftCols}{Vector of character names of columns in data set that should be 30 | included as feature data (row data) in the returned \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 31 | object. (default = \code{NULL})} 32 | 33 | \item{sortIDs}{Whether the output of each method should be merged and sorted using IDs. 34 | See Details for more information. (default = \code{FALSE})} 35 | 36 | \item{keepData}{Whether to store the data as part of the \code{\link[=BenchDesign-class]{BenchDesign}} slot of the 37 | returned \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. If \code{FALSE}, a MD5 hash of the data will be 38 | stored with the \code{\link[=BenchDesign-class]{BenchDesign}} slot. (default = \code{FALSE})} 39 | 40 | \item{catchErrors}{logical whether errors produced by methods during evaluation 41 | should be caught and printed as a message without stopping the entire 42 | build process. (default = \code{TRUE})} 43 | 44 | \item{parallel}{Whether to use parallelization for evaluating each method. 45 | Parallel execution is performed using \pkg{BiocParallel}. Parameters for 46 | parallelization should be specified with \code{\link[BiocParallel]{register}} or 47 | through the \code{BPPARAM} parameter. (default = \code{FALSE})} 48 | 49 | \item{BPPARAM}{Optional \code{BiocParallelParam} instance to be used when 50 | \code{parallel} is \code{TRUE}. If not specified, the default instance from the 51 | parameter registry is used.} 52 | } 53 | \value{ 54 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} object. 55 | } 56 | \description{ 57 | Function to evaluate methods defined in a \code{\link[=BenchDesign-class]{BenchDesign}} on a supplied 58 | data set to generate a \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} of 59 | benchmarking results. In addition to the results of applying each method on the data, the returned 60 | \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} also 61 | includes metadata for the methods 62 | in the \code{colData} of the returned object, metadata for the 63 | data in the \code{rowData}, and session information in the 64 | \code{metadata}. 65 | } 66 | \details{ 67 | Parallelization is performed across methods. Therefore, there is currently no benefit to 68 | specifying more cores than the total number of methods in the 69 | \code{\link[=BenchDesign-class]{BenchDesign}} 70 | object. 71 | 72 | By default, errors thrown by individual methods in the \code{\link[=BenchDesign-class]{BenchDesign}} are caught 73 | during evaluation and handled in a way that allows \code{\link{buildBench}} to continue 74 | running with the other methods. The error is printed as a message, and the corresponding 75 | column in the returned \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}} 76 | object is set to \code{NA}. Since 77 | many benchmarking experiments can be time and computationally intensive, having to rerun 78 | the entire analysis due to a single failed method can be frustrating. Default error catching 79 | was included to alleviate these frustrations. However, if this behavior is not desired, 80 | setting \code{catchErrors = FALSE} will turn off error handling. 81 | 82 | If \code{sortIDs = TRUE}, each method must return a named vector or list. The names will be 83 | used to align the output of each method in the returned \code{\link[=SummarizedBenchmark-class]{SummarizedBenchmark}}. 84 | Missing values 85 | from each method will be set to NA. This can be useful if the different methods return 86 | overlapping, but not identical, results. If \code{truthCols} is also specified, and sorting 87 | by IDs is necessary, rather than specifying \code{sortIDs = TRUE}, specify the string name of a column in 88 | the \code{data} to use to sort the method output to match the order of \code{truthCols}. 89 | 90 | When a method specified in the \code{\link[=BenchDesign-class]{BenchDesign}} does not have a postprocessing function specified 91 | to \code{post =}, the trivial \code{base::identity} function is used as the default postprocessing 92 | function. 93 | } 94 | \examples{ 95 | ## with toy data.frame 96 | df <- data.frame(pval = rnorm(100)) 97 | bench <- BenchDesign(data = df) 98 | 99 | ## add methods 100 | bench <- addMethod(bench, label = "bonf", func = p.adjust, 101 | params = rlang::quos(p = pval, method = "bonferroni")) 102 | bench <- addMethod(bench, label = "BH", func = p.adjust, 103 | params = rlang::quos(p = pval, method = "BH")) 104 | 105 | ## evaluate benchmark experiment 106 | sb <- buildBench(bench) 107 | 108 | ## evaluate benchmark experiment w/ data sepecified 109 | sb <- buildBench(bench, data = df) 110 | 111 | } 112 | \seealso{ 113 | \code{\link{updateBench}} 114 | } 115 | \author{ 116 | Patrick Kimes 117 | } 118 | -------------------------------------------------------------------------------- /docs/reference/BDData-set.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | BDData slot setter — BDData-set • SummarizedBenchmark 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 117 | 118 | 119 |
    120 | 121 |
    122 |
    123 |
    124 |

    BDData slot setter

    125 | Source: R/AllGenerics.R 126 | 127 |
    128 | 129 |
    130 | 131 |

    Method to set BDData slot in objects.

    132 | 133 |
    134 | 135 | 136 |

    Arguments

    137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 147 | 148 | 149 | 150 | 152 | 153 | 154 | 155 | 157 | 158 |
    x

    BenchDesign object.

    i

    character name or integer index of a BDMethod object. 146 | Must be a character name for replacement operations.)

    value

    replacement object, either BDData, BDMethod, BDMethodList 151 | or NULL object.

    ...

    Futher arguments, 156 | perhaps used by methods

    159 | 160 |

    Value

    161 | 162 |

    modified BenchDesign object

    163 | 164 | 165 |
    166 | 179 |
    180 | 181 | 191 |
    192 | 193 | 194 | 195 | 196 | 197 | 198 | --------------------------------------------------------------------------------