├── .Rbuildignore ├── .gitignore ├── DESCRIPTION ├── NAMESPACE ├── R ├── add-to-simulation.R ├── aggregator-class.R ├── component-class.R ├── create.R ├── draws-class.R ├── draws.R ├── evals-class.R ├── evaluate.R ├── examples.R ├── extended-method-class.R ├── get-from-simulation.R ├── import_from.R ├── load.R ├── manage.R ├── method-class.R ├── method-extension-class.R ├── methods.R ├── metric-class.R ├── model-class.R ├── models.R ├── output-class.R ├── parallel-draws.R ├── parallel-methods.R ├── parallel.R ├── plot_eval.R ├── plot_eval_by.R ├── plot_evals.R ├── reference-classes.R ├── simulation-class.R ├── tables.R ├── utils.R └── zzz.R ├── README.Rmd ├── README.md ├── _pkgdown.yml ├── docs ├── .nojekyll ├── 404.html ├── articles │ ├── elastic-net.html │ ├── elastic-net_files │ │ └── figure-html │ │ │ ├── unnamed-chunk-5-1.png │ │ │ ├── unnamed-chunk-6-1.png │ │ │ ├── unnamed-chunk-7-1.png │ │ │ └── unnamed-chunk-9-1.png │ ├── fdr.html │ ├── fdr_files │ │ └── figure-html │ │ │ ├── unnamed-chunk-10-1.png │ │ │ ├── unnamed-chunk-11-1.png │ │ │ ├── unnamed-chunk-12-1.png │ │ │ └── unnamed-chunk-13-1.png │ ├── index.html │ ├── james-stein.html │ ├── james-stein_files │ │ └── figure-html │ │ │ ├── unnamed-chunk-10-1.png │ │ │ ├── unnamed-chunk-12-1.png │ │ │ ├── unnamed-chunk-13-1.png │ │ │ ├── unnamed-chunk-17-1.png │ │ │ ├── unnamed-chunk-18-1.png │ │ │ ├── unnamed-chunk-19-1.png │ │ │ └── unnamed-chunk-6-1.png │ ├── lasso.html │ ├── lasso_files │ │ └── figure-html │ │ │ ├── unnamed-chunk-11-1.png │ │ │ ├── unnamed-chunk-12-1.png │ │ │ ├── unnamed-chunk-16-1.png │ │ │ ├── unnamed-chunk-5-1.png │ │ │ ├── unnamed-chunk-6-1.png │ │ │ └── unnamed-chunk-9-1.png │ └── simulator.html ├── authors.html ├── deps │ ├── bootstrap-5.2.2 │ │ ├── bootstrap.bundle.min.js │ │ ├── bootstrap.bundle.min.js.map │ │ └── bootstrap.min.css │ ├── data-deps.txt │ └── jquery-3.6.0 │ │ ├── jquery-3.6.0.js │ │ ├── jquery-3.6.0.min.js │ │ └── jquery-3.6.0.min.map ├── index.html ├── link.svg ├── pkgdown.js ├── pkgdown.yml ├── reference │ ├── Aggregator-class.html │ ├── Component.html │ ├── Draws-class.html │ ├── DrawsRef-class.html │ ├── Evals-class.html │ ├── EvalsRef-class.html │ ├── ExtendedMethod-class.html │ ├── Method-class.html │ ├── MethodExtension-class.html │ ├── Metric-class.html │ ├── Model-class.html │ ├── ModelRef-class.html │ ├── Output-class.html │ ├── OutputRef-class.html │ ├── Rplot001.png │ ├── Simulation.html │ ├── add.html │ ├── add_bold.html │ ├── add_dref_to_list.html │ ├── add_eref_to_list.html │ ├── add_oref_to_list.html │ ├── aggregate_evals.html │ ├── as.data.frame.Evals.html │ ├── as.data.frame.Model.html │ ├── as.data.frame.listofEvals.html │ ├── as.data.frame.listofModels.html │ ├── cash-Model-method.html │ ├── catsim.html │ ├── create.html │ ├── describe.html │ ├── do_in_parallel.html │ ├── draws.html │ ├── evals.html │ ├── evaluate.html │ ├── evaluate_internal.html │ ├── evaluate_single.html │ ├── figures │ │ └── simulator-hex3.png │ ├── generate_model.html │ ├── get_contents.html │ ├── get_files_not_in_simulations.html │ ├── get_model_indices.html │ ├── get_relative_path.html │ ├── get_simulation_with_all_files.html │ ├── ggplot_eval.html │ ├── grapes-greater-than-grapes.html │ ├── index.html │ ├── load-DrawsRef-method.html │ ├── load-EvalsRef-method.html │ ├── load-ModelRef-method.html │ ├── load-OutputRef-method.html │ ├── load-list-method.html │ ├── load_draws.html │ ├── load_evals.html │ ├── load_libraries_on_cluster.html │ ├── load_model.html │ ├── load_outputs.html │ ├── load_simulation.html │ ├── make_my_example_model.html │ ├── memory_as_string.html │ ├── model.html │ ├── model_names.html │ ├── models_as_data.frame.html │ ├── my_example_loss.html │ ├── my_example_method.html │ ├── new_aggregator.html │ ├── new_extended_method.html │ ├── new_method.html │ ├── new_method_extension.html │ ├── new_metric.html │ ├── new_model.html │ ├── new_simulation.html │ ├── output.html │ ├── outputs_or_evals.html │ ├── plot_eval.html │ ├── plot_eval_by.html │ ├── plot_evals.html │ ├── plus-ExtendedMethod-MethodExtension-method.html │ ├── plus-Method-MethodExtension-method.html │ ├── plus-list-MethodExtension-method.html │ ├── recycle.html │ ├── relabel.html │ ├── rename.html │ ├── run_extendedmethod_single.html │ ├── run_extmethod_parallel.html │ ├── run_method.html │ ├── run_method_parallel.html │ ├── run_method_single.html │ ├── save_simulation.html │ ├── simulate_from_model.html │ ├── simulate_from_model_single.html │ ├── simulate_parallel.html │ ├── subset_evals.html │ ├── subset_models.html │ ├── subset_simulation.html │ └── tabulate_eval.html ├── search.json └── sitemap.xml ├── inst └── CITATION ├── man ├── Aggregator-class.Rd ├── Component.Rd ├── Draws-class.Rd ├── DrawsRef-class.Rd ├── Evals-class.Rd ├── EvalsRef-class.Rd ├── ExtendedMethod-class.Rd ├── Method-class.Rd ├── MethodExtension-class.Rd ├── Metric-class.Rd ├── Model-class.Rd ├── ModelRef-class.Rd ├── Output-class.Rd ├── OutputRef-class.Rd ├── Simulation.Rd ├── add.Rd ├── add_bold.Rd ├── add_dref_to_list.Rd ├── add_eref_to_list.Rd ├── add_oref_to_list.Rd ├── aggregate_evals.Rd ├── as.data.frame.Evals.Rd ├── as.data.frame.Model.Rd ├── as.data.frame.listofEvals.Rd ├── as.data.frame.listofModels.Rd ├── cash-Model-method.Rd ├── catsim.Rd ├── create.Rd ├── describe.Rd ├── do_in_parallel.Rd ├── draws.Rd ├── evals.Rd ├── evaluate.Rd ├── evaluate_internal.Rd ├── evaluate_single.Rd ├── figures │ └── simulator-hex3.png ├── generate_model.Rd ├── get_contents.Rd ├── get_files_not_in_simulations.Rd ├── get_model_indices.Rd ├── get_relative_path.Rd ├── get_simulation_with_all_files.Rd ├── ggplot_eval.Rd ├── grapes-greater-than-grapes.Rd ├── load-DrawsRef-method.Rd ├── load-EvalsRef-method.Rd ├── load-ModelRef-method.Rd ├── load-OutputRef-method.Rd ├── load-list-method.Rd ├── load_draws.Rd ├── load_evals.Rd ├── load_libraries_on_cluster.Rd ├── load_model.Rd ├── load_outputs.Rd ├── load_simulation.Rd ├── make_my_example_model.Rd ├── memory_as_string.Rd ├── model.Rd ├── model_names.Rd ├── models_as_data.frame.Rd ├── my_example_loss.Rd ├── my_example_method.Rd ├── new_aggregator.Rd ├── new_extended_method.Rd ├── new_method.Rd ├── new_method_extension.Rd ├── new_metric.Rd ├── new_model.Rd ├── new_simulation.Rd ├── output.Rd ├── outputs_or_evals.Rd ├── plot_eval.Rd ├── plot_eval_by.Rd ├── plot_evals.Rd ├── plus-ExtendedMethod-MethodExtension-method.Rd ├── plus-Method-MethodExtension-method.Rd ├── plus-list-MethodExtension-method.Rd ├── recycle.Rd ├── relabel.Rd ├── rename.Rd ├── run_extendedmethod_single.Rd ├── run_extmethod_parallel.Rd ├── run_method.Rd ├── run_method_parallel.Rd ├── run_method_single.Rd ├── save_simulation.Rd ├── simulate_from_model.Rd ├── simulate_from_model_single.Rd ├── simulate_parallel.Rd ├── subset_evals.Rd ├── subset_models.Rd ├── subset_simulation.Rd └── tabulate_eval.Rd ├── simulator.Rproj ├── tests ├── testthat.R └── testthat │ ├── test-create.R │ ├── test-draws.R │ ├── test-evaluate.R │ ├── test-method-extension.R │ ├── test-method.R │ ├── test-models.R │ ├── test-parallel.R │ ├── test-regex.R │ ├── test-simulation.R │ ├── test-simulator.files.R │ ├── test-table.R │ └── test-utils.R └── vignettes ├── elastic-net.Rmd ├── elastic-net ├── eval_functions.R ├── main.R ├── method_functions.R └── model_functions.R ├── false-discovery-rate ├── eval_functions.R ├── main.R ├── method_functions.R └── model_functions.R ├── fdr.Rmd ├── james-stein.Rmd ├── james-stein ├── eval_functions.R ├── main.R ├── method_functions.R └── model_functions.R ├── lasso.Rmd ├── lasso ├── eval_functions.R ├── main.R ├── method_functions.R └── model_functions.R └── simulator.Rmd /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | ^README\.Rmd$ 4 | ^README-.*\.png$ 5 | ^_pkgdown\.yml$ 6 | ^docs$ 7 | ^pkgdown$ 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | inst/doc 5 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: simulator 2 | Title: An Engine for Running Simulations 3 | Version: 0.2.4 4 | Date: 2023-02-01 5 | Authors@R: person("Jacob", "Bien", email = "jbien@usc.edu", role = c("aut", "cre")) 6 | Description: A framework for performing simulations such as those common in 7 | methodological statistics papers. The design principles of this package 8 | are described in greater depth in Bien, J. (2016) "The simulator: An Engine 9 | to Streamline Simulations," which is available at . 10 | Depends: 11 | R (>= 3.5.0) 12 | Imports: 13 | methods, 14 | graphics, 15 | grDevices, 16 | stats, 17 | utils, 18 | magrittr 19 | Suggests: 20 | digest, 21 | ggplot2, 22 | knitr, 23 | rmarkdown, 24 | testthat, 25 | glmnet, 26 | mvtnorm 27 | License: GPL-3 28 | LazyData: true 29 | VignetteBuilder: knitr 30 | URL: https://github.com/jacobbien/simulator 31 | BugReports: https://github.com/jacobbien/simulator/issues 32 | Encoding: UTF-8 33 | Collate: 34 | 'evals-class.R' 35 | 'output-class.R' 36 | 'component-class.R' 37 | 'model-class.R' 38 | 'draws-class.R' 39 | 'reference-classes.R' 40 | 'simulation-class.R' 41 | 'add-to-simulation.R' 42 | 'aggregator-class.R' 43 | 'utils.R' 44 | 'create.R' 45 | 'draws.R' 46 | 'evaluate.R' 47 | 'metric-class.R' 48 | 'method-class.R' 49 | 'examples.R' 50 | 'extended-method-class.R' 51 | 'get-from-simulation.R' 52 | 'import_from.R' 53 | 'load.R' 54 | 'manage.R' 55 | 'method-extension-class.R' 56 | 'methods.R' 57 | 'models.R' 58 | 'parallel-draws.R' 59 | 'parallel-methods.R' 60 | 'parallel.R' 61 | 'plot_eval.R' 62 | 'plot_eval_by.R' 63 | 'plot_evals.R' 64 | 'tables.R' 65 | 'zzz.R' 66 | RoxygenNote: 7.2.3 67 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | S3method(as.data.frame,Evals) 4 | S3method(as.data.frame,Model) 5 | S3method(as.data.frame,listofEvals) 6 | S3method(as.data.frame,listofModels) 7 | export("%>%") 8 | export(add) 9 | export(create) 10 | export(describe) 11 | export(draws) 12 | export(evals) 13 | export(evaluate) 14 | export(generate_model) 15 | export(get_contents) 16 | export(get_files_not_in_simulations) 17 | export(get_simulation_with_all_files) 18 | export(load_draws) 19 | export(load_evals) 20 | export(load_evals_from_ref) 21 | export(load_model) 22 | export(load_outputs) 23 | export(load_simulation) 24 | export(make_my_example_model) 25 | export(memory_as_string) 26 | export(model) 27 | export(model_names) 28 | export(my_example_loss) 29 | export(my_example_method) 30 | export(new_aggregator) 31 | export(new_extended_method) 32 | export(new_method) 33 | export(new_method_extension) 34 | export(new_metric) 35 | export(new_model) 36 | export(new_simulation) 37 | export(output) 38 | export(plot_eval) 39 | export(plot_eval_by) 40 | export(plot_evals) 41 | export(relabel) 42 | export(rename) 43 | export(run_method) 44 | export(save_simulation) 45 | export(simulate_from_model) 46 | export(subset_evals) 47 | export(subset_models) 48 | export(subset_simulation) 49 | export(tabulate_eval) 50 | exportClasses(Aggregator) 51 | exportClasses(Component) 52 | exportClasses(Draws) 53 | exportClasses(DrawsRef) 54 | exportClasses(Evals) 55 | exportClasses(EvalsRef) 56 | exportClasses(ExtendedMethod) 57 | exportClasses(Method) 58 | exportClasses(MethodExtension) 59 | exportClasses(Metric) 60 | exportClasses(Model) 61 | exportClasses(ModelRef) 62 | exportClasses(Output) 63 | exportClasses(OutputRef) 64 | exportClasses(Simulation) 65 | exportMethods("$") 66 | exportMethods("+") 67 | exportMethods(load) 68 | importFrom(grDevices,palette) 69 | importFrom(graphics,axis) 70 | importFrom(graphics,boxplot) 71 | importFrom(graphics,legend) 72 | importFrom(graphics,lines) 73 | importFrom(graphics,par) 74 | importFrom(graphics,plot) 75 | importFrom(graphics,points) 76 | importFrom(graphics,segments) 77 | importFrom(graphics,text) 78 | importFrom(graphics,title) 79 | importFrom(magrittr,"%>%") 80 | importFrom(methods,as) 81 | importFrom(methods,callNextMethod) 82 | importFrom(methods,is) 83 | importFrom(methods,new) 84 | importFrom(methods,setClass) 85 | importFrom(methods,setGeneric) 86 | importFrom(methods,setMethod) 87 | importFrom(methods,validObject) 88 | importFrom(parallel,clusterEvalQ) 89 | importFrom(parallel,clusterExport) 90 | importFrom(parallel,makePSOCKcluster) 91 | importFrom(parallel,parLapplyLB) 92 | importFrom(parallel,stopCluster) 93 | importFrom(stats,as.formula) 94 | importFrom(stats,loess) 95 | importFrom(stats,predict) 96 | importFrom(stats,rnorm) 97 | importFrom(stats,sd) 98 | importFrom(utils,installed.packages) 99 | -------------------------------------------------------------------------------- /R/aggregator-class.R: -------------------------------------------------------------------------------- 1 | #' @include evals-class.R 2 | NULL 3 | 4 | check_aggregator <- function(object) { 5 | errors <- character() 6 | if (length(object@label) == 0) 7 | errors <- c(errors, 8 | "Missing \"label\" for object. Make this human-readable.") 9 | args <- names(formals(object@aggregate)) 10 | str <- "aggregate must be a function with argument \"ev\"" 11 | if (length(args) == 1) { 12 | if (args != "ev") 13 | errors <- c(errors, str) 14 | } else if (length(args) != 1) { 15 | errors <- c(errors, str) 16 | } 17 | if (length(errors) == 0) TRUE else errors 18 | } 19 | 20 | 21 | #' An S4 class for aggregating evaluated metrics 22 | #' 23 | #' An object of class \code{Aggregator} consists of a label and a function 24 | #' \code{aggregate} that has a single argument \code{ev} that is a list of 25 | #' length equal to the number of draws. This list consists of the evaluated 26 | #' values of all metrics on a single method for a single model. 27 | #' 28 | #' @slot label a human readable label that will be a prefix to the Eval's label 29 | #' @slot aggregate a function with argument \code{ev} that is a list of 30 | #' length \code{nsim} and returns a scalar. 31 | #' @export 32 | setClass("Aggregator", 33 | representation(label = "character", aggregate = "function"), 34 | validity = check_aggregator) 35 | 36 | #' Create an Aggregator object 37 | #' 38 | #' Creates a new \code{\linkS4class{Aggregator}} object. 39 | #' 40 | #' @param label a human readable label 41 | #' @param aggregate a function with argument \code{ev} that is a list of length 42 | #' equal to the number of draws with each element itself being a named list. 43 | #' Each element of this list corresponds to a metric that has been computed. 44 | #' In particular, given an \code{\linkS4class{Evals}} object \code{o}, 45 | #' \code{aggregate} takes as input \code{o@@evals[[method_name]]} (which is a 46 | #' list of the kind just described). The function aggregate should return a 47 | #' scalar. 48 | #' @export 49 | new_aggregator <- function(label, aggregate) { 50 | new("Aggregator", label = label, aggregate = aggregate) 51 | } 52 | 53 | make_scalar_aggregator <- function(label, metric_name, metric_label, fun) { 54 | new_aggregator(label = paste(label, metric_label), 55 | aggregate = function(ev) { 56 | if (!(metric_name %in% names(ev[[1]]))) 57 | return(NA) 58 | e <- lapply(ev, function(aa) aa[[metric_name]]) 59 | if (all(lapply(e, length) == 1)) 60 | return(fun(unlist(e))) 61 | stop("This aggregator only defined for scalar-", 62 | "valued metrics") 63 | }) 64 | } 65 | 66 | #' Apply aggregator to a list of Evals objects 67 | #' 68 | #' Returns a num_models by num_methods matrix 69 | #' 70 | #' @param evals_list a list of Evals objects 71 | #' @param aggregator object of class Aggregator 72 | aggregate_evals <- function(evals_list, aggregator) { 73 | num_models <- length(evals_list) 74 | method_names <- unique(unlist(lapply(evals_list, 75 | function(e) e@method_name))) 76 | ag <- matrix(NA, num_models, length(method_names)) 77 | for (i in seq(num_models)) { 78 | for (method_name in evals_list[[i]]@method_name) { 79 | j <- which(method_names == method_name) 80 | ag[i, j] <- aggregator@aggregate(evals_list[[i]]@evals[[method_name]]) 81 | } 82 | } 83 | rownames(ag) <- unlist(lapply(evals_list, function(e) e@model_name)) 84 | colnames(ag) <- method_names 85 | ag 86 | } 87 | -------------------------------------------------------------------------------- /R/component-class.R: -------------------------------------------------------------------------------- 1 | check_component <- function(object) { 2 | errors <- character() 3 | errors <- is_valid_component_name(object@name, 4 | sprintf("%s's name", class(object))) 5 | if (length(object@label) == 0) 6 | errors <- c(errors, 7 | "Missing \"label\" for object. Make this human-readable.") 8 | if (length(errors) == 0) TRUE else errors 9 | } 10 | 11 | is_valid_component_name <- function(name, name_of_name, 12 | require_unique = TRUE, 13 | allow_slash = TRUE) { 14 | errors <- character() 15 | if (require_unique) { 16 | if (length(name) != 1) 17 | errors <- c(errors, 18 | sprintf("%s must be of length 1. Make this something short.", 19 | name_of_name)) 20 | } 21 | if (length(name) > 0) { 22 | sub_pattern <- paste0("[[:alnum:]]+((_[[:alnum:]]+)*(-[[:alnum:]]+)*", 23 | "(\\.[[:alnum:]]+)*(_-[[:alnum:]]+)*)*") 24 | # ... of form a or a_b or a-b or a.b (or a-b_c, etc) where a, b, c are 25 | # alphanumeric. Also, allowed to have a_-b allowing for "param_-1.0" 26 | if (allow_slash) { 27 | pattern <- sprintf("^%s(/%s)*$", sub_pattern, sub_pattern) 28 | # ... of form sub_pattern or sub_pattern/sub_pattern etc 29 | str <- paste(name_of_name, 30 | "must be of form a or a/b etc where a and b are", 31 | "alphanumeric strings that can have - or _ or . within.") 32 | if (!all(grepl(pattern, name))) 33 | errors <- c(errors, str) 34 | } else { 35 | pattern <- sprintf("^%s$", sub_pattern) 36 | if (!all(grepl(pattern, name))) 37 | errors <- c(errors, 38 | paste0(name_of_name, 39 | " must be an alphanumeric string (which can also", 40 | " have - or _ within).")) 41 | } 42 | } 43 | errors 44 | } 45 | 46 | #' An S4 class representing a component of the simulator. 47 | #' 48 | #' This is a virtual class. 49 | #' 50 | #' @slot name a short name identifier. Must be alphanumeric. 51 | #' @slot label a longer, human readable label that can have other characters 52 | #' such as spaces, hyphens, etc. 53 | #' 54 | #' @rdname Component 55 | #' @export 56 | setClass("Component", 57 | representation( 58 | name = "character", # shortname identifier 59 | label = "character", # human readable label 60 | "VIRTUAL" # this is a virtual class 61 | ), 62 | validity = check_component 63 | ) 64 | 65 | setMethod("show", "Component", function(object) { 66 | validObject(object) 67 | cat(paste0(class(object), " Component"), fill = TRUE) 68 | cat(paste0(" name: ", object@name), fill = TRUE) 69 | cat(paste0(" label: ", object@label), fill = TRUE) 70 | }) 71 | -------------------------------------------------------------------------------- /R/draws-class.R: -------------------------------------------------------------------------------- 1 | #' @include model-class.R 2 | NULL 3 | 4 | check_draws <- function(object) { 5 | errors <- check_component(object) 6 | if (length(errors) == 1) 7 | if(errors == TRUE) errors <- character() 8 | if (length(object@index) < 1) errors <- c(errors, 9 | "index must have length >= 1.") 10 | if (any(round(object@index) != object@index)) errors <- c(errors, 11 | "index must be an integer.") 12 | 13 | if (length(object@draws) < 1) { 14 | errors <- c(errors, "draws must be nonempty.") 15 | } else { 16 | str <- "incorrectly named elements of draws See documentation." 17 | # should be of format ri.j where i is in index and j starts at 1. 18 | pattern <- "^r([[:digit:]]+)[.]([[:digit:]]+)$" 19 | i <- as.numeric(gsub(pattern, "\\1", names(object@draws))) 20 | j <- as.numeric(gsub(pattern, "\\2", names(object@draws))) 21 | if (any(is.na(i))) errors <- c(errors, str) 22 | if (any(sort(unique(i)) != sort(object@index))) 23 | errors <- c(errors, "index does not match elements in draws list.") 24 | } 25 | if (length(errors) == 0) TRUE else errors 26 | } 27 | 28 | 29 | #' An S4 class representing the random draws from a Model object. 30 | #' 31 | #' An object of class Draws represents the randomly drawn simulated data that is 32 | #' generated when \code{\link{simulate_from_model}} is called on an object of 33 | #' class Model. In particular, it contains a named list of \code{nsim} 34 | #' simulated draws from a model object. The Model object's \code{simulate} 35 | #' function populates this list. 36 | #' 37 | #' This class inherits from the \code{\linkS4class{Component}} class. 38 | #' 39 | #' @slot name a short name identifier. Must be alphanumeric. Should use 40 | #' the name of the Model object that generated it. 41 | #' @slot label a longer, human readable label that indicates what has been 42 | #' randomly drawn. 43 | #' @slot draws a list with \code{nsim} elements as created by calling 44 | #' the \code{simulate} function of a Model object. This is a named 45 | #' list with each element labeled as \code{ri.j} where \code{i} is the 46 | #' \code{index} and \code{j} ranges from \code{1} to \code{nsim}. The 47 | #' names are assigned by \code{\link{simulate_from_model}}. 48 | #' @slot index an integer-valued numeric that indicates which block of random 49 | #' draws this is 50 | #' 51 | #' @export 52 | setClass("Draws", representation(index = "numeric", draws = "list"), 53 | contains = "Component", validity = check_draws) 54 | 55 | setMethod("show", "Draws", function(object) { 56 | validObject(object) 57 | callNextMethod() 58 | catsim(" (Add @draws to end of this object to see what was simulated.)", 59 | fill = TRUE) 60 | }) 61 | -------------------------------------------------------------------------------- /R/examples.R: -------------------------------------------------------------------------------- 1 | #' @include model-class.R method-class.R metric-class.R 2 | NULL 3 | 4 | #' Make My Example Model 5 | #' 6 | #' This function is used in the examples. It returns a 7 | #' \code{\linkS4class{Model}} object. In particular, it represents 8 | #' \code{n} i.i.d. draws from a normal with mean 2 and variance 1. 9 | #' 10 | #' @param n number of i.i.d. draws 11 | #' 12 | #' @export 13 | #' @seealso \code{\link{my_example_method}} \code{\link{my_example_loss}} 14 | make_my_example_model <- function(n) { 15 | new_model(name = "normal-data", 16 | label = sprintf("Normal (n = %s)", n), 17 | params = list(n = n, mu = 2), 18 | simulate = function(n, mu, nsim) { 19 | # this function must return a list of length nsim 20 | x <- matrix(rnorm(n * nsim), n, nsim) 21 | x <- mu + x # true mean is mu 22 | return(split(x, col(x))) # make each col its own list element 23 | }) 24 | } 25 | 26 | #' My Example Method 27 | #' 28 | #' This \code{\linkS4class{Method}} object is used in the examples. It is 29 | #' the sample mean of the data. 30 | #' @export 31 | #' @seealso \code{\link{make_my_example_model}} \code{\link{my_example_loss}} 32 | my_example_method <- new_method("my-method", "My Example Method", 33 | method = function(model, draw) { 34 | list(fit = mean(draw)) 35 | }) 36 | 37 | #' My Example Loss 38 | #' 39 | #' This \code{\linkS4class{Metric}} object is used in the examples. It is 40 | #' squared error loss. 41 | #' @export 42 | #' @seealso \code{\link{make_my_example_model}} \code{\link{my_example_loss}} 43 | my_example_loss <- new_metric("myloss", "My Example Loss", 44 | metric = function(model, out) { 45 | return((model$mu - out$fit)^2) 46 | }) 47 | 48 | -------------------------------------------------------------------------------- /R/extended-method-class.R: -------------------------------------------------------------------------------- 1 | #' @include method-class.R 2 | NULL 3 | 4 | check_extended_method <- function(object) { 5 | errors <- check_component(object) 6 | errors <- is_valid_component_name(object@name, "name", allow_slash = FALSE) 7 | if (length(errors) == 1) 8 | if(errors == TRUE) errors <- character() 9 | args <- names(formals(object@extended_method)) 10 | str <- paste("extended_method must be a function with arguments", 11 | "\"model\", \"draw\", \"out\", and \"base_method\".") 12 | if (length(args) != 4 || any(args != c("model", "draw", "out", 13 | "base_method"))) 14 | errors <- c(errors, str) 15 | if (length(object@base_method) != 1) 16 | errors <- c(errors, "base_method must be a list of length 1") 17 | if (!(class(object@base_method[[1]]) %in% c("Method", "ExtendedMethod"))) { 18 | str <- "base_method[[1]] must be of class Method or ExtendedMethod" 19 | errors <- c(errors, str) 20 | } 21 | if (length(errors) == 0) TRUE else errors 22 | } 23 | 24 | #setClassUnion("Method_or_ExtendedMethod", c("Method", "ExtendedMethod")) 25 | 26 | #' An S4 class representing the extension of a method 27 | #' 28 | #' An object of class \code{ExtendedMethod} is like a 29 | #' \code{\linkS4class{Method}} except it uses the output of another method in 30 | #' addition to the \code{\linkS4class{Model}} and 31 | #' \code{\linkS4class{Draws}}. We can also form chains of 32 | #' \code{ExtendedMethod}'s, in which one \code{ExtendedMethod} is taken to be 33 | #' the "\code{base_method}" of a subsequent \code{ExtendedMethod}. This means 34 | #' that the latter \code{ExtendedMethod} would use the output of the former 35 | #' \code{ExtendedMethod}. 36 | #' 37 | #' While one can create an \code{\linkS4class{ExtendedMethod}} from scratch, 38 | #' typically it will be cleaner to write a \code{MethodExtension} object 39 | #' and then use the addition operator: 40 | #' \code{my_extended_method = my_base_method + my_method_extension}. For 41 | #' example, if \code{my_base_method} is the lasso, \code{my_method_extension} 42 | #' might be cross-validation, and the resulting \code{my_extended_method} would 43 | #' be the lasso with tuning parameter chosen by cross-validation. The advantage 44 | #' is that if we have several methods, we only have to write the 45 | #' cross-validation \code{MethodExtension} object once. 46 | #' 47 | #' For an example in which one has a chain of \code{ExtendedMethod}'s, consider 48 | #' the lasso example in which we have a \code{MethodExtension} called, say, 49 | #' \code{refit}, which takes the nonzeros from the lasso's output and 50 | #' performs least squares on these selected variables. Let \code{cv} be another 51 | #' \code{MethodExtension}. Then, \code{refitted_lasso = lasso + refit} is 52 | #' an \code{ExtendedMethod} and \code{refitted_lasso + cv} is as well. 53 | #' 54 | #' This class inherits from the \code{\linkS4class{Component}} class. 55 | #' 56 | #' @slot name a short name identifier. Must be alphanumeric. 57 | #' @slot label a longer, human readable label that can have other characters 58 | #' such as spaces, hyphens, etc. 59 | #' @slot base_method a list of length 1 containing the object of class 60 | #' \code{\linkS4class{Method}} or \code{\linkS4class{ExtendedMethod}} 61 | #' that is being extended 62 | #' @slot extended_method a function with arguments "model", "draw", "out", and 63 | #' "base_method". 64 | #' @export 65 | setClass("ExtendedMethod", 66 | #representation(base_method = "Method_or_ExtendedMethod", 67 | # this doesn't work because it depends on ExtendedMethod being defined 68 | representation(base_method = "list", # this must be of length one and 69 | # contain either a Method or an ExtendedMethod 70 | extended_method = "function"), 71 | contains = "Component", validity = check_extended_method) 72 | 73 | #' Create an ExtendedMethod object 74 | #' 75 | #' Creates a new \code{\linkS4class{ExtendedMethod}} object. 76 | #' 77 | #' @param name a short name identifier. Must be alphanumeric. 78 | #' @param label a longer, human readable label that can have other characters 79 | #' such as spaces, hyphens, etc. 80 | #' @param base_method the object of class \code{\linkS4class{Method}} or 81 | #' of class \code{\linkS4class{Method}} that is being extended 82 | #' @param extended_method a function with arguments "model", "draw", "out", and 83 | #' "base_method". 84 | #' @export 85 | new_extended_method <- function(name, label, base_method, extended_method) { 86 | new("ExtendedMethod", name = name, label = label, 87 | base_method = list(base_method), 88 | extended_method = extended_method) 89 | } 90 | -------------------------------------------------------------------------------- /R/import_from.R: -------------------------------------------------------------------------------- 1 | 2 | #' This is not magrittr's pipe (actually it is) 3 | #' @name %>% 4 | #' @importFrom magrittr %>% 5 | #' @export 6 | #' @keywords internal 7 | NULL 8 | 9 | #' @importFrom methods setClass setGeneric setMethod as is 10 | #' callNextMethod new validObject 11 | #' @importFrom grDevices palette 12 | #' @importFrom graphics axis boxplot legend lines par plot points segments text 13 | #' title 14 | #' @importFrom stats as.formula loess predict sd rnorm 15 | #' @importFrom utils installed.packages 16 | #' @importFrom parallel makePSOCKcluster clusterExport parLapplyLB stopCluster 17 | #' clusterExport clusterEvalQ 18 | NULL 19 | -------------------------------------------------------------------------------- /R/method-class.R: -------------------------------------------------------------------------------- 1 | #' @include component-class.R output-class.R 2 | NULL 3 | 4 | check_method <- function(object) { 5 | errors <- check_component(object) 6 | errors <- is_valid_component_name(object@name, "name", allow_slash = FALSE) 7 | if (length(errors) == 1) 8 | if(errors == TRUE) errors <- character() 9 | args <- names(formals(object@method)) 10 | str <- "method must be a function with arguments \"model\" and \"draw\"." 11 | if (!all((c("model", "draw") %in% args))) 12 | errors <- c(errors, str) 13 | if (length(errors) == 0) TRUE else errors 14 | } 15 | 16 | 17 | #' An S4 class representing a method to be run by simulator. 18 | #' 19 | #' An object of class \code{Method} consists of a name, label, and a function 20 | #' \code{method} that takes arguments model and draw. A draw refers 21 | #' to a single element of the list in an object of class \code{\linkS4class{Draws}}. 22 | #' 23 | #' This class inherits from the \code{\linkS4class{Component}} class. 24 | #' 25 | #' @slot name a short name identifier. Must be alphanumeric. 26 | #' @slot label a longer, human readable label that can have other characters 27 | #' such as spaces, hyphens, etc. 28 | #' @slot settings (optional) a list of "settings" for the method (e.g., tuning 29 | #' parameters or related information that might distinguish two otherwise 30 | #' identical methods). 31 | #' @slot method a function that has arguments "model", "draw" 32 | #' and (optionally) names matching elements within \code{names(settings)} 33 | #' @export 34 | setClass("Method", representation(settings = "list", method = "function"), 35 | contains = "Component", validity = check_method) 36 | 37 | #' Create a Method object 38 | #' 39 | #' Creates a new \code{\linkS4class{Method}} object. 40 | #' 41 | #' @param name a short name identifier. Must be alphanumeric. 42 | #' @param label a longer, human readable label that can have other characters 43 | #' such as spaces, hyphens, etc. 44 | #' @param method a function that has arguments "model", "draw" 45 | #' and (optionally) names matching elements within \code{names(settings)} 46 | #' @param settings (optional) a list of "settings" for the method (e.g., tuning 47 | #' parameters or related information that might distinguish two otherwise 48 | #' identical methods). 49 | #' @export 50 | new_method <- function(name, label, method, settings = list()) { 51 | new("Method", name = name, label = label, settings = settings, method = method) 52 | } 53 | 54 | setMethod("show", "Method", function(object) { 55 | validObject(object) 56 | callNextMethod() 57 | catsim(" (Add @method to end of this object to see function.)", 58 | fill = TRUE) 59 | if (length(object@settings) > 0) 60 | catsim(" (Add @settings to end of this object to see list of settings.)", 61 | fill = TRUE) 62 | }) 63 | -------------------------------------------------------------------------------- /R/method-extension-class.R: -------------------------------------------------------------------------------- 1 | #' @include extended-method-class.R 2 | NULL 3 | 4 | check_method_extension <- function(object) { 5 | errors <- check_component(object) 6 | errors <- is_valid_component_name(object@name, "name", allow_slash = FALSE) 7 | if (length(errors) == 1) 8 | if(errors == TRUE) errors <- character() 9 | args <- names(formals(object@method_extension)) 10 | str <- paste("method_extension must be a function with arguments", 11 | "\"model\", \"draw\", \"out\", and \"base_method\".") 12 | if (length(args) != 4 || any(args != c("model", "draw", "out", 13 | "base_method"))) 14 | errors <- c(errors, str) 15 | if (length(errors) == 0) TRUE else errors 16 | } 17 | 18 | #' An S4 class used to create an extended version of a method 19 | #' 20 | #' An object of class \code{MethodExtension} when added to a \code{Method} 21 | #' creates a \code{\linkS4class{ExtendedMethod}}. 22 | #' 23 | #' This class inherits from the \code{\linkS4class{Component}} class. 24 | #' 25 | #' @slot name a short name identifier. Must be alphanumeric. 26 | #' @slot label a longer, human readable label that can have other characters 27 | #' such as spaces, hyphens, etc. 28 | #' @slot method_extension a function with arguments "model", "draw", "out", and 29 | #' "base_method". This will become the function \code{extended_method} 30 | #' in the \code{ExtendedMethod} object that is created. 31 | #' @export 32 | setClass("MethodExtension", 33 | representation(method_extension = "function"), 34 | contains = "Component", validity = check_method_extension) 35 | 36 | #' Create an object that can be used to make an extended version of a method 37 | #' 38 | #' Creates an object of class \code{MethodExtension}, which when added to a 39 | #' \code{Method} creates an \code{\linkS4class{ExtendedMethod}}. 40 | #' 41 | #' This class inherits from the \code{\linkS4class{Component}} class. 42 | #' 43 | #' @param name a short name identifier. Must be alphanumeric. 44 | #' @param label a longer, human readable label that can have other characters 45 | #' such as spaces, hyphens, etc. 46 | #' @param method_extension a function with arguments "model", "draw", "out", and 47 | #' "base_method". This will become the function \code{extended_method} 48 | #' in the \code{ExtendedMethod} object that is created. 49 | #' @export 50 | new_method_extension <- function(name, label, method_extension) { 51 | new("MethodExtension", name = name, label = label, 52 | method_extension = method_extension) 53 | } 54 | 55 | #' Create an ExtendedMethod from a Method and MethodExtension 56 | #' 57 | #' @param e1 an object of class \code{\linkS4class{Method}} 58 | #' @param e2 an object of class \code{\linkS4class{MethodExtension}} 59 | #' @export 60 | setMethod("+", signature(e1 = "Method", e2 = "MethodExtension"), 61 | function(e1, e2) { 62 | new_extended_method(paste0(e1@name, "_", e2@name), 63 | paste0(e1@label, " ", e2@label), 64 | base_method = e1, 65 | extended_method = e2@method_extension) 66 | }) 67 | 68 | #' Create an ExtendedMethod from an ExtendedMethod and MethodExtension 69 | #' 70 | #' @param e1 an object of class \code{\linkS4class{ExtendedMethod}} 71 | #' @param e2 an object of class \code{\linkS4class{MethodExtension}} 72 | #' @export 73 | setMethod("+", signature(e1 = "ExtendedMethod", e2 = "MethodExtension"), 74 | function(e1, e2) { 75 | new_extended_method(paste0(e1@name, "_", e2@name), 76 | paste0(e1@label, " ", e2@label), 77 | base_method = e1, 78 | extended_method = e2@method_extension) 79 | }) 80 | 81 | #' Create a list of ExtendedMethod from a list of Methods and a MethodExtension 82 | #' 83 | #' @param e1 a list of objects of class \code{\linkS4class{Method}} or of class 84 | #' \code{\linkS4class{ExtendedMethod}} 85 | #' @param e2 an object of class \code{\linkS4class{MethodExtension}} 86 | #' @export 87 | setMethod("+", signature(e1 = "list", e2 = "MethodExtension"), 88 | function(e1, e2) { 89 | stopifnot(lapply(e1, class) %in% c("Method", "ExtendedMethod")) 90 | lapply(e1, function(m) { 91 | new_extended_method(paste0(m@name, "_", e2@name), 92 | paste0(m@label, " ", e2@label), 93 | base_method = m, 94 | extended_method = e2@method_extension) 95 | }) 96 | }) 97 | 98 | -------------------------------------------------------------------------------- /R/metric-class.R: -------------------------------------------------------------------------------- 1 | #' @include component-class.R output-class.R 2 | NULL 3 | 4 | check_metric <- function(object) { 5 | errors <- check_component(object) 6 | name_is_alphanumeric <- grepl("^[[:alnum:]_]+$", object@name) 7 | if (!name_is_alphanumeric) 8 | errors <- c(errors, 9 | "metric name must be alphanumeric (and can have underscores).") 10 | if (length(errors) == 1) 11 | if(errors == TRUE) errors <- character() 12 | args <- names(formals(object@metric)) 13 | str <- paste("metric must be a function with arguments \"model\" and", 14 | "\"out\" (and optionally \"draw\")") 15 | if (length(args) == 2) { 16 | if (!all(sort(args) == c("model", "out"))) 17 | errors <- c(errors, str) 18 | } 19 | else if (length(args) == 3) { 20 | if (!all(sort(args) == c("draw", "model", "out"))) 21 | errors <- c(errors, str) 22 | } 23 | else 24 | errors <- c(errors, str) 25 | if (length(errors) == 0) TRUE else errors 26 | } 27 | 28 | 29 | #' An S4 class representing an evaluation metric to be used by simulator. 30 | #' 31 | #' An object of class \code{Metric} consists of a name, label, and a function 32 | #' \code{metric} that takes arguments \code{model} (of class 33 | #' \code{\linkS4class{Model}}) and \code{out} (of class \code{\linkS4class{Output}}), which 34 | #' is the output of a method. 35 | #' 36 | #' This class inherits from the \code{\linkS4class{Component}} class. 37 | #' 38 | #' @slot name a short name identifier. Must be alphanumeric. 39 | #' @slot label a longer, human readable label that can have other characters 40 | #' such as spaces, hyphens, etc. 41 | #' @slot metric a function with arguments "model" and "out" (and optionally 42 | #' "draw") 43 | #' @export 44 | setClass("Metric", representation(metric = "function"), 45 | contains = "Component", validity = check_metric) 46 | 47 | #' Create a Metric object 48 | #' 49 | #' Creates a new \code{\linkS4class{Metric}} object. 50 | #' 51 | #' @param name a short name identifier. Must be alphanumeric. 52 | #' @param label a longer, human readable label that can have other characters 53 | #' such as spaces, hyphens, etc. 54 | #' @param metric a function with arguments "model" and "out" (and optionally 55 | #' "draw") 56 | #' @export 57 | new_metric <- function(name, label, metric) { 58 | new("Metric", name = name, label = label, metric = metric) 59 | } 60 | 61 | setMethod("show", "Metric", function(object) { 62 | validObject(object) 63 | callNextMethod() 64 | catsim(" (Add @metric to end of this object to see function.)", 65 | fill = TRUE) 66 | }) 67 | -------------------------------------------------------------------------------- /R/output-class.R: -------------------------------------------------------------------------------- 1 | is_valid_rij_list <- function(out, index) { 2 | if (length(out) < 1) { 3 | errors <- c(errors, "out must be nonempty.") 4 | } else { 5 | str <- "incorrectly named elements of out. See documentation." 6 | # should be of format ri.j where i is in index and j starts at 1. 7 | pattern <- "^r([[:digit:]]+)[.]([[:digit:]]+)$" 8 | i <- as.numeric(gsub(pattern, "\\1", names(out))) 9 | j <- as.numeric(gsub(pattern, "\\2", names(out))) 10 | if (any(is.na(i))) errors <- c(errors, str) 11 | if (any(sort(unique(i)) != sort(index))) 12 | errors <- c(errors, "index does not match elements in out list.") 13 | } 14 | if (!all(unlist(lapply(out, is.list)))) 15 | errors <- c(errors, "out$ri.j should be a list.") 16 | nams <- lapply(out, function(r) names(r)) 17 | if (!all(unlist(lapply(nams, function(nam) identical(nam, nams[[1]]))))) 18 | errors <- c(errors, "all out$ri.j must have same elements.") 19 | } 20 | 21 | check_output <- function(object) { 22 | errors <- is_valid_component_name(object@model_name, "model_name") 23 | errors <- c(errors, 24 | is_valid_component_name(object@method_name, "method_name", 25 | allow_slash = FALSE)) 26 | 27 | if (length(object@index) < 1) 28 | errors <- c(errors, "index must be of length >= 1.") 29 | else if (any(object@index != round(object@index))) 30 | errors <- c(errors, "index must be an integer-valued numeric.") 31 | if (length(object@method_label) != 1) 32 | errors <- c(errors, "method_label must be of length 1.") 33 | errors <- c(errors, is_valid_rij_list(object@out, object@index)) 34 | if (length(errors) == 0) TRUE else errors 35 | } 36 | 37 | 38 | #' An S4 class representing the output of a method run by simulator. 39 | #' 40 | #' An object of class \code{Output} consists of information to identify the 41 | #' model, draws, and method objects this output was derived from. It also has 42 | #' a list called \code{out}, which is where the output of the method is stored. 43 | #' 44 | #' @slot model_name the name of the \code{\linkS4class{Model}} object this output is 45 | #' derived from. 46 | #' @slot index the index of the \code{\linkS4class{Draws}} object this output is 47 | #' derived from. 48 | #' @slot method_name the name of the \code{\linkS4class{Method}} object this output is 49 | #' derived from. 50 | #' @slot method_label the label of the \code{\linkS4class{Method}} object this output 51 | #' is derived from. 52 | #' @slot out a named list with each element labeled as \code{ri.j} where 53 | #' \code{i} is the \code{index} and \code{j} ranges from \code{1} to 54 | #' \code{nsim}. Element \code{out$ri.j} is output of method 55 | #' \code{method_name} on random draw \code{ri.j}. 56 | #' @export 57 | setClass("Output", representation(model_name = "character", 58 | method_name = "character", 59 | method_label = "character", 60 | index = "numeric", 61 | out = "list"), 62 | validity = check_output) 63 | 64 | setMethod("show", "Output", function(object) { 65 | validObject(object) 66 | cat(paste0("Output Component"), fill = TRUE) 67 | cat(paste0(" model_name: ", object@model_name), fill = TRUE) 68 | cat(paste0(" index: ", paste(object@index, collapse = ", ")), fill = TRUE) 69 | if (length(object@index) == 1) 70 | cat(paste0(" nsim: ", length(object@out)), fill = TRUE) 71 | else 72 | cat(paste0(" nsim (total): ", length(object@out)), fill = TRUE) 73 | cat(paste0(" method_name: ", object@method_name), fill = TRUE) 74 | cat(paste0(" method_label: ", object@method_label), fill = TRUE) 75 | cat(paste0(" out: ", paste(names(object@out[[1]]), collapse = ", ")), 76 | fill = TRUE) 77 | catsim(" (Add @out to end of this object to see what method(s) returned.)", 78 | fill = TRUE) 79 | }) 80 | -------------------------------------------------------------------------------- /R/parallel-draws.R: -------------------------------------------------------------------------------- 1 | #' Simulate from a model in parallel. 2 | #' 3 | #' This is an internal function. Draws are done in chunks labeled 4 | #' by indices and of size determined by nsim. Users should call the wrapper 5 | #' function \code{\link{simulate_from_model}}. 6 | #' 7 | #' @param model_ref object of class \code{\linkS4class{ModelRef}} 8 | #' @param nsim number of simulations to be conducted on each chunk. Vector of 9 | #' same length as \code{index} 10 | #' @param index a vector of positive integer indices. Allows simulations to be 11 | #' carried out in chunks. Each chunk gets a separate RNG stream, 12 | #' meaning that the results will be identical whether we run these in 13 | #' parallel or sequentially. 14 | #' @param seeds a list of \code{length(index)} L'Ecuyer-CMRG seed vectors. 15 | #' Each should be from a separate stream. In particular, starting from 16 | #' the seed used to generate the model object, seeds[i] should be the 17 | #' result of calling \code{\link[parallel]{nextRNGStream}} index[i] 18 | #' times. 19 | #' @param socket_names (quoting from \code{\link[parallel]{makePSOCKcluster}} 20 | #' "either a character vector of host names on which to run the worker 21 | #' copies of R, or a positive integer (in which case that number of 22 | #' copies is run on localhost)." 23 | #' @param libraries character vector of R packages that will be needed on the 24 | #' slaves. 25 | #' @param save_locally if TRUE, then files will be saved on slaves. If FALSE, 26 | #' they will be saved on master. 27 | simulate_parallel <- function(model_ref, nsim, index, seeds, 28 | socket_names, libraries, save_locally = TRUE) { 29 | stopifnot(length(index) == length(nsim)) 30 | # this is function to be run on each slave: 31 | function_to_do <- function(model_ref, nsim, index, seed) { 32 | model <- load(model_ref) 33 | simulate_from_model_single(model, nsim, index, seed) 34 | } 35 | # make list where params1[[i]] are the arguments to pass to 36 | # function_to_do for i-th index 37 | params1 <- lapply(seq(length(index)), 38 | function(i) list(model_ref = model_ref, 39 | nsim = nsim[i], index = index[i], 40 | seed = seeds[[i]])) 41 | # this is function to use when saving info in d to file (whether it be on 42 | # slave or master): 43 | save_to_file <- function(draws, rng, time, out_dir, model_ref, index, nsim) { 44 | save_draws_to_file(out_dir = out_dir, model_ref = model_ref, index = index, 45 | nsim = nsim, draws = draws, rng = rng, time = time[1]) 46 | } 47 | # parameters to be passed to save_to_file other than d 48 | md <- get_model_dir_and_file(model_ref@dir, model_ref@name) 49 | params2 <- lapply(seq(length(index)), 50 | function(i) list(out_dir = md$dir, 51 | model_ref = model_ref, 52 | index = index[i], 53 | nsim = nsim[i])) 54 | do_in_parallel(function_to_do = function_to_do, 55 | function_params = params1, 56 | save_to_file = save_to_file, 57 | save_params = params2, 58 | socket_names = socket_names, 59 | libraries = libraries, 60 | save_locally = save_locally) 61 | } 62 | -------------------------------------------------------------------------------- /R/parallel.R: -------------------------------------------------------------------------------- 1 | #' Load necessary libraries on a cluster 2 | #' @keywords internal 3 | load_libraries_on_cluster <- function(cl, libs) { 4 | libs <- unique(libs) 5 | stopifnot(is.character(libs)) 6 | parallel::clusterExport(cl, "libs", envir = environment()) 7 | parallel::clusterEvalQ(cl, sapply(libs, 8 | function(pkgnam) { do.call("library", 9 | list(pkgnam))})) 10 | } 11 | 12 | check_parallel_list <- function(parallel) { 13 | if (!is.list(parallel)) stop("parallel must be a list.") 14 | parallel_args <- c("socket_names", "libraries", "save_locally") 15 | if (length(setdiff(names(parallel), parallel_args)) > 0) { 16 | args <- paste(parallel_args, collapse = ", ") 17 | stop("parallel must be a list containing only ", args) 18 | } 19 | if (!("socket_names" %in% names(parallel))) 20 | stop("parallel must be a list containing \"socket_names\".") 21 | if ("libraries" %in% names(parallel)) 22 | stopifnot(is.character(parallel$libraries)) 23 | if ("save_locally" %in% names(parallel)) 24 | stopifnot(is.logical(parallel$save_locally)) 25 | } 26 | 27 | #' Do a function in parallel. 28 | #' 29 | #' This is an internal function. The function_to_do is done in parallel 30 | #' and its output is saved either on slave (locally) or on master. 31 | #' 32 | #' @param function_to_do this is the function that will be done in parallel 33 | #' @param function_params a list where \code{function_params[[i]]} is 34 | #' a named list of parameters to be passed to \code{function_to_do} for 35 | #' job \code{i} 36 | #' @param save_to_file function that saves stuff to file 37 | #' @param save_params a list where \code{save_params[[i]]} 38 | #' is a named list of parameters to be passed to \code{save_to_file} for 39 | #' job \code{i}. Each \code{save_params[[i]]} must include \code{out_dir}, 40 | #' which is location where file is to be saved. 41 | #' @param socket_names quoting from \code{\link[parallel]{makePSOCKcluster}}: 42 | #' "either a character vector of host names on which to run the worker 43 | #' copies of R, or a positive integer (in which case that number of 44 | #' copies is run on localhost)." 45 | #' @param libraries character vector of R packages that will be needed on the 46 | #' slaves. 47 | #' @param save_locally if TRUE, then files will be saved on slaves. If FALSE, 48 | #' they will be saved on master. 49 | #' @keywords internal 50 | do_in_parallel <- function(function_to_do, function_params, 51 | save_to_file, save_params, socket_names, 52 | libraries, save_locally = TRUE) { 53 | njobs <- length(function_params) 54 | for (i in seq(njobs)) stopifnot("out_dir" %in% names(save_params[[i]])) 55 | inner_wrapper <- function(i) { 56 | do.call("function_to_do", function_params[[i]]) 57 | } 58 | if (save_locally) { 59 | wrapper <- function(i) { 60 | # create directory on slave if it doesn't yet exist: 61 | if (!dir.exists(save_params[[i]]$out_dir)) 62 | dir.create(save_params[[i]]$out_dir, recursive = TRUE) 63 | d <- inner_wrapper(i) 64 | if (any(names(d) %in% names(save_params[[i]]))) 65 | stop("save_params must not contain any elements with 66 | same name as list output of function_to_do.") 67 | # save this on slave 68 | file <- do.call("save_to_file", c(d, save_params[[i]])) 69 | return(file) 70 | } 71 | } else { 72 | wrapper <- inner_wrapper 73 | } 74 | if (getOption("simulator.slave_stdout_stderr_to_master")) 75 | cl <- parallel::makePSOCKcluster(names = socket_names, outfile = "") 76 | else 77 | cl <- parallel::makePSOCKcluster(names = socket_names) 78 | if (!("simulator" %in% libraries)) libraries <- c("simulator", libraries) 79 | 80 | tryCatch({ 81 | load_libraries_on_cluster(cl, libraries) 82 | # export the parameters that will be needed by function_to_do on slaves 83 | parallel::clusterExport(cl, varlist = c("function_params", "save_params"), 84 | envir = environment()) 85 | parallel::clusterExport(cl, varlist = ls(envir = globalenv())) 86 | out <- parallel::parLapplyLB(cl, seq(njobs), wrapper) 87 | }, finally = { 88 | # regardless of whether an error occurs, do close cluster. 89 | message("Shutting down cluster.") 90 | parallel::stopCluster(cl)}) 91 | if (save_locally) { 92 | # out is a list of file names on slave where output was saved 93 | refs <- out 94 | } else { 95 | # out is a list of outputs of function_to_do 96 | # we now save these to file (on master) 97 | refs <- list() 98 | for (i in seq(njobs)) 99 | refs[[i]] <- do.call("save_to_file", c(out[[i]], save_params[[i]])) 100 | } 101 | catsim("..Created", as.character(refs), "in parallel.", fill = TRUE) 102 | if (save_locally) catsim("(saved on slaves)", fill = TRUE) 103 | invisible(refs) 104 | } 105 | -------------------------------------------------------------------------------- /R/utils.R: -------------------------------------------------------------------------------- 1 | 2 | remove_slash <- function(s) { 3 | # if ends with slash, remove it 4 | n <- nchar(s) 5 | ifelse(substr(s, n, n) == "/", substr(s, 1, n - 1), s) 6 | } 7 | 8 | get_model_dir_and_file <- function(dir, model_name, simulator.files = NULL) { 9 | dir <- remove_slash(dir) 10 | if (is.null(simulator.files)) simulator.files <- getOption("simulator.files") 11 | model_dir <- file.path(dir, simulator.files, model_name) 12 | model_file <- file.path(model_dir, "model.Rdata") 13 | if (!file.exists(model_dir)) 14 | stop("Could not find models directory ", model_dir, call. = FALSE) 15 | if (!file.info(model_dir)$isdir) 16 | stop(model_dir, " is not a directory.",call. = FALSE) 17 | if (!file.exists(model_file)) 18 | stop("Could not find file ", model_file, call. = FALSE) 19 | list(dir = model_dir, file = model_file) 20 | } 21 | 22 | #' Concatenate and print for the simulator 23 | #' 24 | #' For internal use. This calls \code{\link{cat}} only when 25 | #' \code{getOption("simulator.verbose")}. 26 | #' @param ... arguments to be passed to \code{\link{cat}} 27 | catsim <- function(...) { 28 | if (getOption("simulator.verbose")) cat(...) 29 | } 30 | 31 | 32 | #' Get relative path 33 | #' 34 | #' Given a base path and a specific path, returns a string str such that 35 | #' file.path(base_path, str) is the same location as path. 36 | #' 37 | #' @param base_path the base path 38 | #' @param path a specific path 39 | get_relative_path <- function(base_path, path) { 40 | b <- strsplit(normalizePath(base_path, mustWork = FALSE, winslash = "/"), 41 | split = "/")[[1]] 42 | p <- strsplit(normalizePath(path, mustWork = FALSE, winslash = "/"), 43 | split = "/")[[1]] 44 | len <- min(length(b), length(p)) 45 | ncommon <- max(which(b[1:len] == p[1:len])) 46 | str <- rep("..", length(b) - ncommon) 47 | if (length(p) > ncommon) 48 | str <- c(str, p[(ncommon + 1):length(p)]) 49 | return(paste0(str, collapse = "/")) 50 | } 51 | 52 | #' Write memory in human readable way 53 | #' 54 | #' @param memory_in_bytes the amount of memory in Bytes. 55 | #' @export 56 | memory_as_string <- function(memory_in_bytes) { 57 | units <- c("Bytes", "KB", "MB", "GB", "TB") 58 | ii <- floor(log(memory_in_bytes, 1000)) 59 | if (ii >= length(units)) stop("Really? That's an enormous simulation!") 60 | if (ii < 0) stop("Negative memory? How strange.") 61 | paste(round(memory_in_bytes / 1000^ii, 2), units[ii+1]) 62 | } 63 | -------------------------------------------------------------------------------- /R/zzz.R: -------------------------------------------------------------------------------- 1 | .onLoad <- function(libname, pkgname) { 2 | # most of this function is copied from Hadley Wickham's dplyr/zzz.R 3 | op <- options() 4 | op.simulator <- list( 5 | simulator.verbose = TRUE, # controls catsim 6 | simulator.ndecimal = 5, # how long can a decimal be before "digest" called? 7 | simulator.files = "files", # directory created in user supplied "dir" 8 | simulator.slave_stdout_stderr_to_master = FALSE, # useful for debugging 9 | simulator.color_palette = c("#000000", "#e41a1c", "#377eb8", "#4daf4a", 10 | "#984ea3", "#ff7f00", "#ffff33"), 11 | # from http://colorbrewer2.org/ print-friendly, 6 classes (added black) 12 | simulator.plot_evals.index.name = "plot_evals_x_axis_index_variable", 13 | # this is for when plot_evals is called with metric_name_x = NULL 14 | simulator.plot_evals.index.label = " " 15 | ) 16 | toset <- !(names(op.simulator) %in% names(op)) 17 | if(any(toset)) options(op.simulator[toset]) 18 | 19 | invisible() 20 | } 21 | -------------------------------------------------------------------------------- /README.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | output: github_document 3 | --- 4 | 5 | 6 | 7 | ```{r, echo = FALSE} 8 | knitr::opts_chunk$set( 9 | collapse = TRUE, 10 | comment = "#>", 11 | fig.path = "README-" 12 | ) 13 | ``` 14 | 15 | # The Simulator 16 | 17 | The [simulator](https://jacobbien.github.io/simulator/) is an R package that streamlines the process of performing simulations by creating a common infrastructure that can be easily used and reused across projects. The paper [The Simulator: An Engine to Streamline Simulations](https://arxiv.org/abs/1607.00021) discusses the simulator in greater depth. 18 | 19 | ## Installation 20 | 21 | The most up-to-date version of the simulator is on this github page and can be installed using the [devtools](https://cran.r-project.org/package=devtools) R package (if not already installed, open R and type `install.packages("devtools")`). To install simulator, type 22 | 23 | ```{r, eval = FALSE} 24 | devtools::install_github("jacobbien/simulator") 25 | ``` 26 | in R. This installs `simulator` from github. 27 | 28 | ## Vignettes 29 | 30 | The [Getting Started](https://jacobbien.github.io/simulator/articles/simulator.html) vignette walks you through setting up your first simulation with the simulator. 31 | 32 | The best way to get a sense of how to use the simulator is to look at examples. There are several vignettes that demonstrate how the simulator can be used to conduct simulations for some of the most famous statistical methods. 33 | 34 | 1) [Lasso](https://jacobbien.github.io/simulator/articles/lasso.html) vignette: Explains basics, including the magrittr pipe and making plots and tables. Also demonstrates some more advanced features such as writing method extensions (such as refitting the result of the lasso or performing cross-validation). 35 | 2) [James-Stein](https://jacobbien.github.io/simulator/articles/james-stein.html) vignette: Shows how to step into specific parts of the simulation for troubleshooting your code. 36 | 3) [Elastic net](https://jacobbien.github.io/simulator/articles/elastic-net.html) vignette: Shows how we can work with a sequence of methods that are identical except for a parameter that varies 37 | 4) [Benjamini-Hochberg](https://jacobbien.github.io/simulator/articles/fdr.html) vignette: Shows how we can load a preexisting simulation and add more random draws without having to rerun anything. It also shows how one can have multiple simulation objects that point to overlapping sets of results. 38 | 39 | 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # The Simulator 5 | 6 | The [simulator](https://jacobbien.github.io/simulator/) is an R package 7 | that streamlines the process of performing simulations by creating a 8 | common infrastructure that can be easily used and reused across 9 | projects. The paper [The Simulator: An Engine to Streamline 10 | Simulations](https://arxiv.org/abs/1607.00021) discusses the simulator 11 | in greater depth. 12 | 13 | ## Installation 14 | 15 | The most up-to-date version of the simulator is on this github page and 16 | can be installed using the 17 | [devtools](https://cran.r-project.org/package=devtools) R package (if 18 | not already installed, open R and type `install.packages("devtools")`). 19 | To install simulator, type 20 | 21 | ``` r 22 | devtools::install_github("jacobbien/simulator") 23 | ``` 24 | 25 | in R. This installs `simulator` from github. 26 | 27 | ## Vignettes 28 | 29 | The [Getting 30 | Started](https://jacobbien.github.io/simulator/articles/simulator.html) 31 | vignette walks you through setting up your first simulation with the 32 | simulator. 33 | 34 | The best way to get a sense of how to use the simulator is to look at 35 | examples. There are several vignettes that demonstrate how the simulator 36 | can be used to conduct simulations for some of the most famous 37 | statistical methods. 38 | 39 | 1) [Lasso](https://jacobbien.github.io/simulator/articles/lasso.html) 40 | vignette: Explains basics, including the magrittr pipe and making 41 | plots and tables. Also demonstrates some more advanced features such 42 | as writing method extensions (such as refitting the result of the 43 | lasso or performing cross-validation). 44 | 2) [James-Stein](https://jacobbien.github.io/simulator/articles/james-stein.html) 45 | vignette: Shows how to step into specific parts of the simulation 46 | for troubleshooting your code. 47 | 3) [Elastic 48 | net](https://jacobbien.github.io/simulator/articles/elastic-net.html) 49 | vignette: Shows how we can work with a sequence of methods that are 50 | identical except for a parameter that varies 51 | 4) [Benjamini-Hochberg](https://jacobbien.github.io/simulator/articles/fdr.html) 52 | vignette: Shows how we can load a preexisting simulation and add 53 | more random draws without having to rerun anything. It also shows 54 | how one can have multiple simulation objects that point to 55 | overlapping sets of results. 56 | -------------------------------------------------------------------------------- /_pkgdown.yml: -------------------------------------------------------------------------------- 1 | url: ~ 2 | template: 3 | bootstrap: 5 4 | navbar: 5 | structure: 6 | left: [intro, reference, articles, tutorials, news] 7 | right: [search, github] 8 | components: 9 | articles: 10 | text: Examples 11 | menu: 12 | - text: "The Lasso with the Simulator" 13 | href: articles/lasso.html 14 | - text: "James-Stein with the Simulator" 15 | href: articles/james-stein.html 16 | - text: "The Elastic Net with the Simulator" 17 | href: articles/elastic-net.html 18 | - text: "Benjamini-Hochberg with the Simulator" 19 | href: articles/fdr.html 20 | authors: 21 | Jacob Bien: 22 | href: http://faculty.marshall.usc.edu/jacob-bien/ 23 | reference: 24 | - title: Getting started 25 | desc: A useful way to get started is to call this function from the console. 26 | contents: 27 | - create 28 | - title: Running a simulation 29 | desc: These are the most common functions you'll use when running a simulation. 30 | contents: 31 | - new_simulation 32 | - generate_model 33 | - simulate_from_model 34 | - run_method 35 | - evaluate 36 | - title: Defining simulation components 37 | desc: These are the functions you'll use when defining the components of a simulation. 38 | contents: 39 | - new_model 40 | - new_method 41 | - new_metric 42 | - new_method_extension 43 | - new_aggregator 44 | - title: Loading saved files from simulation 45 | desc: These are functions for loading files that were created when you ran your simulation. These allow you to look at simulation outputs at various intermediate stages of the simulation process. 46 | contents: 47 | - load_simulation 48 | - subset_simulation 49 | - model 50 | - draws 51 | - output 52 | - evals 53 | 54 | - title: Making plots and tables 55 | desc: Functions for making the most common plots and tables in a simulation. For more customized plots, you can instead export the simulation outputs to a data.frame and then make your own plots. 56 | contents: 57 | - plot_eval 58 | - plot_eval_by 59 | - plot_evals 60 | - tabulate_eval 61 | - aggregate_evals 62 | 63 | - title: Managing simulation files 64 | desc: Functions for checking and manipulating the files that have been stored by the simulator. This is for more advanced use. 65 | contents: 66 | - get_contents 67 | - describe 68 | - get_files_not_in_simulations 69 | - get_simulation_with_all_files 70 | - add 71 | - save_simulation 72 | 73 | - title: internal 74 | contents: 75 | - Aggregator-class 76 | - Component-class 77 | - Draws-class 78 | - DrawsRef-class 79 | - Evals-class 80 | - EvalsRef-class 81 | - ExtendedMethod-class 82 | - Method-class 83 | - MethodExtension-class 84 | - Metric-class 85 | - Model-class 86 | - ModelRef-class 87 | - Output-class 88 | - OutputRef-class 89 | - Simulation-class 90 | - add_bold 91 | - as.data.frame.Evals 92 | - as.data.frame.Model 93 | - as.data.frame.listofEvals 94 | - as.data.frame.listofModels 95 | - $,Model-method 96 | - catsim 97 | - evaluate_internal 98 | - evaluate_single 99 | - get_model_indices 100 | - get_relative_path 101 | - load,DrawsRef-method 102 | - load,EvalsRef-method 103 | - load,ModelRef-method 104 | - load,OutputRef-method 105 | - load,list-method 106 | - load_draws 107 | - load_evals 108 | - load_model 109 | - make_my_example_model 110 | - memory_as_string 111 | - model_names 112 | - models_as_data.frame 113 | - my_example_loss 114 | - my_example_method 115 | - new_extended_method 116 | - +,ExtendedMethod,MethodExtension-method 117 | - +,Method,MethodExtension-method 118 | - +,list,MethodExtension-method 119 | - recycle 120 | - relabel 121 | - rename 122 | - run_extendedmethod_single 123 | - run_method_single 124 | - simulate_from_model_single 125 | - simulate_parallel 126 | - subset_evals 127 | - subset_models 128 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/articles/elastic-net_files/figure-html/unnamed-chunk-5-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/elastic-net_files/figure-html/unnamed-chunk-5-1.png -------------------------------------------------------------------------------- /docs/articles/elastic-net_files/figure-html/unnamed-chunk-6-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/elastic-net_files/figure-html/unnamed-chunk-6-1.png -------------------------------------------------------------------------------- /docs/articles/elastic-net_files/figure-html/unnamed-chunk-7-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/elastic-net_files/figure-html/unnamed-chunk-7-1.png -------------------------------------------------------------------------------- /docs/articles/elastic-net_files/figure-html/unnamed-chunk-9-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/elastic-net_files/figure-html/unnamed-chunk-9-1.png -------------------------------------------------------------------------------- /docs/articles/fdr_files/figure-html/unnamed-chunk-10-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/fdr_files/figure-html/unnamed-chunk-10-1.png -------------------------------------------------------------------------------- /docs/articles/fdr_files/figure-html/unnamed-chunk-11-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/fdr_files/figure-html/unnamed-chunk-11-1.png -------------------------------------------------------------------------------- /docs/articles/fdr_files/figure-html/unnamed-chunk-12-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/fdr_files/figure-html/unnamed-chunk-12-1.png -------------------------------------------------------------------------------- /docs/articles/fdr_files/figure-html/unnamed-chunk-13-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/fdr_files/figure-html/unnamed-chunk-13-1.png -------------------------------------------------------------------------------- /docs/articles/james-stein_files/figure-html/unnamed-chunk-10-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/james-stein_files/figure-html/unnamed-chunk-10-1.png -------------------------------------------------------------------------------- /docs/articles/james-stein_files/figure-html/unnamed-chunk-12-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/james-stein_files/figure-html/unnamed-chunk-12-1.png -------------------------------------------------------------------------------- /docs/articles/james-stein_files/figure-html/unnamed-chunk-13-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/james-stein_files/figure-html/unnamed-chunk-13-1.png -------------------------------------------------------------------------------- /docs/articles/james-stein_files/figure-html/unnamed-chunk-17-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/james-stein_files/figure-html/unnamed-chunk-17-1.png -------------------------------------------------------------------------------- /docs/articles/james-stein_files/figure-html/unnamed-chunk-18-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/james-stein_files/figure-html/unnamed-chunk-18-1.png -------------------------------------------------------------------------------- /docs/articles/james-stein_files/figure-html/unnamed-chunk-19-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/james-stein_files/figure-html/unnamed-chunk-19-1.png -------------------------------------------------------------------------------- /docs/articles/james-stein_files/figure-html/unnamed-chunk-6-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/james-stein_files/figure-html/unnamed-chunk-6-1.png -------------------------------------------------------------------------------- /docs/articles/lasso_files/figure-html/unnamed-chunk-11-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/lasso_files/figure-html/unnamed-chunk-11-1.png -------------------------------------------------------------------------------- /docs/articles/lasso_files/figure-html/unnamed-chunk-12-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/lasso_files/figure-html/unnamed-chunk-12-1.png -------------------------------------------------------------------------------- /docs/articles/lasso_files/figure-html/unnamed-chunk-16-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/lasso_files/figure-html/unnamed-chunk-16-1.png -------------------------------------------------------------------------------- /docs/articles/lasso_files/figure-html/unnamed-chunk-5-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/lasso_files/figure-html/unnamed-chunk-5-1.png -------------------------------------------------------------------------------- /docs/articles/lasso_files/figure-html/unnamed-chunk-6-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/lasso_files/figure-html/unnamed-chunk-6-1.png -------------------------------------------------------------------------------- /docs/articles/lasso_files/figure-html/unnamed-chunk-9-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/articles/lasso_files/figure-html/unnamed-chunk-9-1.png -------------------------------------------------------------------------------- /docs/deps/data-deps.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /docs/link.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 8 | 12 | 13 | -------------------------------------------------------------------------------- /docs/pkgdown.yml: -------------------------------------------------------------------------------- 1 | pandoc: 2.19.2 2 | pkgdown: 2.0.7 3 | pkgdown_sha: ~ 4 | articles: 5 | elastic-net: elastic-net.html 6 | fdr: fdr.html 7 | james-stein: james-stein.html 8 | lasso: lasso.html 9 | simulator: simulator.html 10 | last_built: 2023-02-02T07:50Z 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/Rplot001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/reference/Rplot001.png -------------------------------------------------------------------------------- /docs/reference/figures/simulator-hex3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/docs/reference/figures/simulator-hex3.png -------------------------------------------------------------------------------- /inst/CITATION: -------------------------------------------------------------------------------- 1 | bibentry(bibtype = "article", 2 | title = "The {simulator}: An Engine to Streamline Simulations", 3 | author = person("Jacob", "Bien"), 4 | year = {2016}, 5 | url = "https://arxiv.org/abs/1607.00021", 6 | journal = "arXiv:1607.00021") 7 | -------------------------------------------------------------------------------- /man/Aggregator-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/aggregator-class.R 3 | \docType{class} 4 | \name{Aggregator-class} 5 | \alias{Aggregator-class} 6 | \title{An S4 class for aggregating evaluated metrics} 7 | \description{ 8 | An object of class \code{Aggregator} consists of a label and a function 9 | \code{aggregate} that has a single argument \code{ev} that is a list of 10 | length equal to the number of draws. This list consists of the evaluated 11 | values of all metrics on a single method for a single model. 12 | } 13 | \section{Slots}{ 14 | 15 | \describe{ 16 | \item{\code{label}}{a human readable label that will be a prefix to the Eval's label} 17 | 18 | \item{\code{aggregate}}{a function with argument \code{ev} that is a list of 19 | length \code{nsim} and returns a scalar.} 20 | }} 21 | 22 | -------------------------------------------------------------------------------- /man/Component.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/component-class.R 3 | \docType{class} 4 | \name{Component-class} 5 | \alias{Component-class} 6 | \title{An S4 class representing a component of the simulator.} 7 | \description{ 8 | This is a virtual class. 9 | } 10 | \section{Slots}{ 11 | 12 | \describe{ 13 | \item{\code{name}}{a short name identifier. Must be alphanumeric.} 14 | 15 | \item{\code{label}}{a longer, human readable label that can have other characters 16 | such as spaces, hyphens, etc.} 17 | }} 18 | 19 | -------------------------------------------------------------------------------- /man/Draws-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/draws-class.R 3 | \docType{class} 4 | \name{Draws-class} 5 | \alias{Draws-class} 6 | \title{An S4 class representing the random draws from a Model object.} 7 | \description{ 8 | An object of class Draws represents the randomly drawn simulated data that is 9 | generated when \code{\link{simulate_from_model}} is called on an object of 10 | class Model. In particular, it contains a named list of \code{nsim} 11 | simulated draws from a model object. The Model object's \code{simulate} 12 | function populates this list. 13 | } 14 | \details{ 15 | This class inherits from the \code{\linkS4class{Component}} class. 16 | } 17 | \section{Slots}{ 18 | 19 | \describe{ 20 | \item{\code{name}}{a short name identifier. Must be alphanumeric. Should use 21 | the name of the Model object that generated it.} 22 | 23 | \item{\code{label}}{a longer, human readable label that indicates what has been 24 | randomly drawn.} 25 | 26 | \item{\code{draws}}{a list with \code{nsim} elements as created by calling 27 | the \code{simulate} function of a Model object. This is a named 28 | list with each element labeled as \code{ri.j} where \code{i} is the 29 | \code{index} and \code{j} ranges from \code{1} to \code{nsim}. The 30 | names are assigned by \code{\link{simulate_from_model}}.} 31 | 32 | \item{\code{index}}{an integer-valued numeric that indicates which block of random 33 | draws this is} 34 | }} 35 | 36 | -------------------------------------------------------------------------------- /man/DrawsRef-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/reference-classes.R 3 | \docType{class} 4 | \name{DrawsRef-class} 5 | \alias{DrawsRef-class} 6 | \title{An S4 class representing a reference to an object of class Draws.} 7 | \description{ 8 | This identifies the necessary information to locate a saved object 9 | of class \code{\linkS4class{Draws}}. 10 | } 11 | \section{Slots}{ 12 | 13 | \describe{ 14 | \item{\code{dir}}{directory where the directory \code{getOption("simulator.files")} 15 | is that contains the referenced \code{\linkS4class{Model}} object} 16 | 17 | \item{\code{model_name}}{name of the referenced \code{\linkS4class{Model}} object} 18 | 19 | \item{\code{index}}{the index of the referenced \code{\linkS4class{Draws}} object. Can 20 | alternately be a vector of such indices.} 21 | 22 | \item{\code{simulator.files}}{simulator functions will use 23 | \code{getOption("simulator.files")} if simulator.files not provided.} 24 | }} 25 | 26 | -------------------------------------------------------------------------------- /man/Evals-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/evals-class.R 3 | \docType{class} 4 | \name{Evals-class} 5 | \alias{Evals-class} 6 | \title{An S4 class representing the evaluation of a metric run by simulator.} 7 | \description{ 8 | An object of class \code{Evals} consists of information to identify the 9 | model, draws, method, and metric objects this output was derived from. It 10 | also has a list called \code{evals}, which is where the output of the metric 11 | is stored. Currently, the labels of all these objects are also included so 12 | that plot functions can use human-readable labels without requiring 13 | re-loading these. 14 | } 15 | \section{Slots}{ 16 | 17 | \describe{ 18 | \item{\code{model_name}}{the name of the \code{\linkS4class{Model}} object this output is 19 | derived from.} 20 | 21 | \item{\code{model_label}}{the label of the \code{\linkS4class{Model}} object this output is 22 | derived from.} 23 | 24 | \item{\code{index}}{the index of the \code{\linkS4class{Draws}} object this output is 25 | derived from.} 26 | 27 | \item{\code{method_name}}{the name of the \code{\linkS4class{Method}} object this output is 28 | derived from.} 29 | 30 | \item{\code{method_label}}{the label of the \code{\linkS4class{Method}} object this output 31 | is derived from.} 32 | 33 | \item{\code{metric_name}}{the name of the \code{\linkS4class{Metric}} object this output is 34 | derived from.} 35 | 36 | \item{\code{metric_label}}{the label of the \code{\linkS4class{Metric}} object this output 37 | is derived from.} 38 | 39 | \item{\code{evals}}{a named list with each element labeled by a method_name 40 | each evals[[m]] is itself a named list with each element labeled 41 | as \code{ri.j} where \code{i} is the \code{index} and \code{j} ranges 42 | from \code{1} to \code{nsim}. Element \code{out$ri.j} is output of 43 | metric \code{metric_name} on random draw \code{ri.j}.} 44 | }} 45 | 46 | \seealso{ 47 | \code{\link{evaluate}} \code{\link{as.data.frame.Evals}} 48 | } 49 | -------------------------------------------------------------------------------- /man/EvalsRef-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/reference-classes.R 3 | \docType{class} 4 | \name{EvalsRef-class} 5 | \alias{EvalsRef-class} 6 | \title{An S4 class representing a reference to an object of class Evals} 7 | \description{ 8 | This identifies the necessary information to locate a saved object 9 | of class \code{\linkS4class{Evals}}. Note that \code{metric_names} is not 10 | needed to identify an Evals object since Evals objects combine all metrics 11 | together into a single file and object. 12 | } 13 | \section{Slots}{ 14 | 15 | \describe{ 16 | \item{\code{dir}}{directory where the directory \code{getOption("simulator.files")} 17 | is that contains the referenced \code{\linkS4class{Model}} object} 18 | 19 | \item{\code{model_name}}{name of the referenced \code{\linkS4class{Model}} object} 20 | 21 | \item{\code{index}}{the index of the referenced \code{\linkS4class{Draws}} object.} 22 | 23 | \item{\code{method_name}}{the name of the \code{\linkS4class{Method}} object this output is 24 | derived from.} 25 | 26 | \item{\code{out_loc}}{a length-1 character vector that gives location 27 | (relative to model's path) that method outputs are stored.This can be 28 | useful for staying organized when multiple simulations are based on 29 | the same Model and Draws objects.} 30 | 31 | \item{\code{simulator.files}}{simulator functions will use 32 | \code{getOption("simulator.files")} if simulator.files not provided.} 33 | }} 34 | 35 | -------------------------------------------------------------------------------- /man/ExtendedMethod-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/extended-method-class.R 3 | \docType{class} 4 | \name{ExtendedMethod-class} 5 | \alias{ExtendedMethod-class} 6 | \title{An S4 class representing the extension of a method} 7 | \description{ 8 | An object of class \code{ExtendedMethod} is like a 9 | \code{\linkS4class{Method}} except it uses the output of another method in 10 | addition to the \code{\linkS4class{Model}} and 11 | \code{\linkS4class{Draws}}. We can also form chains of 12 | \code{ExtendedMethod}'s, in which one \code{ExtendedMethod} is taken to be 13 | the "\code{base_method}" of a subsequent \code{ExtendedMethod}. This means 14 | that the latter \code{ExtendedMethod} would use the output of the former 15 | \code{ExtendedMethod}. 16 | } 17 | \details{ 18 | While one can create an \code{\linkS4class{ExtendedMethod}} from scratch, 19 | typically it will be cleaner to write a \code{MethodExtension} object 20 | and then use the addition operator: 21 | \code{my_extended_method = my_base_method + my_method_extension}. For 22 | example, if \code{my_base_method} is the lasso, \code{my_method_extension} 23 | might be cross-validation, and the resulting \code{my_extended_method} would 24 | be the lasso with tuning parameter chosen by cross-validation. The advantage 25 | is that if we have several methods, we only have to write the 26 | cross-validation \code{MethodExtension} object once. 27 | 28 | For an example in which one has a chain of \code{ExtendedMethod}'s, consider 29 | the lasso example in which we have a \code{MethodExtension} called, say, 30 | \code{refit}, which takes the nonzeros from the lasso's output and 31 | performs least squares on these selected variables. Let \code{cv} be another 32 | \code{MethodExtension}. Then, \code{refitted_lasso = lasso + refit} is 33 | an \code{ExtendedMethod} and \code{refitted_lasso + cv} is as well. 34 | 35 | This class inherits from the \code{\linkS4class{Component}} class. 36 | } 37 | \section{Slots}{ 38 | 39 | \describe{ 40 | \item{\code{name}}{a short name identifier. Must be alphanumeric.} 41 | 42 | \item{\code{label}}{a longer, human readable label that can have other characters 43 | such as spaces, hyphens, etc.} 44 | 45 | \item{\code{base_method}}{a list of length 1 containing the object of class 46 | \code{\linkS4class{Method}} or \code{\linkS4class{ExtendedMethod}} 47 | that is being extended} 48 | 49 | \item{\code{extended_method}}{a function with arguments "model", "draw", "out", and 50 | "base_method".} 51 | }} 52 | 53 | -------------------------------------------------------------------------------- /man/Method-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/method-class.R 3 | \docType{class} 4 | \name{Method-class} 5 | \alias{Method-class} 6 | \title{An S4 class representing a method to be run by simulator.} 7 | \description{ 8 | An object of class \code{Method} consists of a name, label, and a function 9 | \code{method} that takes arguments model and draw. A draw refers 10 | to a single element of the list in an object of class \code{\linkS4class{Draws}}. 11 | } 12 | \details{ 13 | This class inherits from the \code{\linkS4class{Component}} class. 14 | } 15 | \section{Slots}{ 16 | 17 | \describe{ 18 | \item{\code{name}}{a short name identifier. Must be alphanumeric.} 19 | 20 | \item{\code{label}}{a longer, human readable label that can have other characters 21 | such as spaces, hyphens, etc.} 22 | 23 | \item{\code{settings}}{(optional) a list of "settings" for the method (e.g., tuning 24 | parameters or related information that might distinguish two otherwise 25 | identical methods).} 26 | 27 | \item{\code{method}}{a function that has arguments "model", "draw" 28 | and (optionally) names matching elements within \code{names(settings)}} 29 | }} 30 | 31 | -------------------------------------------------------------------------------- /man/MethodExtension-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/method-extension-class.R 3 | \docType{class} 4 | \name{MethodExtension-class} 5 | \alias{MethodExtension-class} 6 | \title{An S4 class used to create an extended version of a method} 7 | \description{ 8 | An object of class \code{MethodExtension} when added to a \code{Method} 9 | creates a \code{\linkS4class{ExtendedMethod}}. 10 | } 11 | \details{ 12 | This class inherits from the \code{\linkS4class{Component}} class. 13 | } 14 | \section{Slots}{ 15 | 16 | \describe{ 17 | \item{\code{name}}{a short name identifier. Must be alphanumeric.} 18 | 19 | \item{\code{label}}{a longer, human readable label that can have other characters 20 | such as spaces, hyphens, etc.} 21 | 22 | \item{\code{method_extension}}{a function with arguments "model", "draw", "out", and 23 | "base_method". This will become the function \code{extended_method} 24 | in the \code{ExtendedMethod} object that is created.} 25 | }} 26 | 27 | -------------------------------------------------------------------------------- /man/Metric-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/metric-class.R 3 | \docType{class} 4 | \name{Metric-class} 5 | \alias{Metric-class} 6 | \title{An S4 class representing an evaluation metric to be used by simulator.} 7 | \description{ 8 | An object of class \code{Metric} consists of a name, label, and a function 9 | \code{metric} that takes arguments \code{model} (of class 10 | \code{\linkS4class{Model}}) and \code{out} (of class \code{\linkS4class{Output}}), which 11 | is the output of a method. 12 | } 13 | \details{ 14 | This class inherits from the \code{\linkS4class{Component}} class. 15 | } 16 | \section{Slots}{ 17 | 18 | \describe{ 19 | \item{\code{name}}{a short name identifier. Must be alphanumeric.} 20 | 21 | \item{\code{label}}{a longer, human readable label that can have other characters 22 | such as spaces, hyphens, etc.} 23 | 24 | \item{\code{metric}}{a function with arguments "model" and "out" (and optionally 25 | "draw")} 26 | }} 27 | 28 | -------------------------------------------------------------------------------- /man/Model-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/model-class.R 3 | \docType{class} 4 | \name{Model-class} 5 | \alias{Model-class} 6 | \title{An S4 class representing the model component of the simulator.} 7 | \description{ 8 | An object of class Model specifies the statistical model. In particular, 9 | all parameters are specified in addition to a function called \code{simulate} 10 | that allows one to draw random samples from this model. 11 | } 12 | \details{ 13 | To get parameters stored in a \code{Model} object, a shortcut for 14 | \code{my_model@params$my_parameter} is \code{my_model$my_parameter}. 15 | 16 | This class inherits from the \code{\linkS4class{Component}} class. 17 | } 18 | \section{Slots}{ 19 | 20 | \describe{ 21 | \item{\code{name}}{a short name identifier. Must be alphanumeric (though -, _, and 22 | / are allowed as long as they are not at the start or end of name.} 23 | 24 | \item{\code{label}}{a longer, human readable label that can have other characters 25 | such as spaces, hyphens, etc.} 26 | 27 | \item{\code{params}}{a list that contains the Model object's parameters} 28 | 29 | \item{\code{simulate}}{a function that has arguments \code{nsim} and names 30 | matching elements within \code{names(params)}. It returns a list of 31 | length nsim, where each element of the list represents a random draw 32 | from the \code{Model} object.} 33 | }} 34 | 35 | -------------------------------------------------------------------------------- /man/ModelRef-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/reference-classes.R 3 | \docType{class} 4 | \name{ModelRef-class} 5 | \alias{ModelRef-class} 6 | \title{An S4 class representing a reference to an object of class Model.} 7 | \description{ 8 | This identifies the necessary information to locate a saved object 9 | of class \code{\linkS4class{Model}}. 10 | } 11 | \section{Slots}{ 12 | 13 | \describe{ 14 | \item{\code{dir}}{directory where the directory "files" is that contains the 15 | referenced \code{\linkS4class{Model}} object} 16 | 17 | \item{\code{name}}{a short name identifier.} 18 | 19 | \item{\code{label}}{a longer, human readable label that can have other characters} 20 | 21 | \item{\code{simulator.files}}{simulator functions will use 22 | \code{getOption("simulator.files")} if simulator.files not provided.} 23 | }} 24 | 25 | -------------------------------------------------------------------------------- /man/Output-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/output-class.R 3 | \docType{class} 4 | \name{Output-class} 5 | \alias{Output-class} 6 | \title{An S4 class representing the output of a method run by simulator.} 7 | \description{ 8 | An object of class \code{Output} consists of information to identify the 9 | model, draws, and method objects this output was derived from. It also has 10 | a list called \code{out}, which is where the output of the method is stored. 11 | } 12 | \section{Slots}{ 13 | 14 | \describe{ 15 | \item{\code{model_name}}{the name of the \code{\linkS4class{Model}} object this output is 16 | derived from.} 17 | 18 | \item{\code{index}}{the index of the \code{\linkS4class{Draws}} object this output is 19 | derived from.} 20 | 21 | \item{\code{method_name}}{the name of the \code{\linkS4class{Method}} object this output is 22 | derived from.} 23 | 24 | \item{\code{method_label}}{the label of the \code{\linkS4class{Method}} object this output 25 | is derived from.} 26 | 27 | \item{\code{out}}{a named list with each element labeled as \code{ri.j} where 28 | \code{i} is the \code{index} and \code{j} ranges from \code{1} to 29 | \code{nsim}. Element \code{out$ri.j} is output of method 30 | \code{method_name} on random draw \code{ri.j}.} 31 | }} 32 | 33 | -------------------------------------------------------------------------------- /man/OutputRef-class.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/reference-classes.R 3 | \docType{class} 4 | \name{OutputRef-class} 5 | \alias{OutputRef-class} 6 | \title{An S4 class representing a reference to an object of class Output.} 7 | \description{ 8 | This identifies the necessary information to locate a saved object 9 | of class \code{\linkS4class{Output}}. 10 | } 11 | \section{Slots}{ 12 | 13 | \describe{ 14 | \item{\code{dir}}{directory where the directory \code{getOption("simulator.files")} 15 | is that contains the referenced \code{\linkS4class{Model}} object} 16 | 17 | \item{\code{model_name}}{name of the referenced \code{\linkS4class{Model}} object} 18 | 19 | \item{\code{index}}{the index of the referenced \code{\linkS4class{Draws}} object. Can 20 | alternately be a vector of such indices.} 21 | 22 | \item{\code{method_name}}{the name of the \code{\linkS4class{Method}} object this output is 23 | derived from.} 24 | 25 | \item{\code{out_loc}}{a length-1 character vector that gives location 26 | (relative to model's path) that method outputs are stored.This can be 27 | useful for staying organized when multiple simulations are based on 28 | the same Model and Draws objects.} 29 | 30 | \item{\code{simulator.files}}{simulator functions will use 31 | \code{getOption("simulator.files")} if simulator.files not provided.} 32 | }} 33 | 34 | -------------------------------------------------------------------------------- /man/Simulation.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/simulation-class.R 3 | \docType{class} 4 | \name{Simulation-class} 5 | \alias{Simulation-class} 6 | \title{An S4 class representing a simulation.} 7 | \description{ 8 | A simulation is a set of references to simulator objects that have been 9 | saved to file. The DrawsRef, OutputRef, and EvalsRef objects are organized 10 | by model into separate lists. 11 | } 12 | \details{ 13 | When a reference ref is added to a simulation sim, ref@dir is changed so 14 | that the referenced file is located at file.path(sim@dir, ref@dir). 15 | } 16 | \section{Slots}{ 17 | 18 | \describe{ 19 | \item{\code{name}}{a short name identifier. Must be an alphanumeric (but can also 20 | have - or _ within} 21 | 22 | \item{\code{label}}{a longer, human readable label that can have other characters 23 | such as spaces, hyphens, etc.} 24 | 25 | \item{\code{dir}}{name of the directory where directory named "files" exists.} 26 | 27 | \item{\code{model_refs}}{a list of \code{\linkS4class{ModelRef}} objects} 28 | 29 | \item{\code{draws_refs}}{a list of lists of \code{\linkS4class{DrawsRef}} objects} 30 | 31 | \item{\code{output_refs}}{a list of lists of \code{\linkS4class{OutputRef}} objects} 32 | 33 | \item{\code{evals_refs}}{a list of lists of \code{\linkS4class{EvalsRef}} objects} 34 | }} 35 | 36 | -------------------------------------------------------------------------------- /man/add.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/add-to-simulation.R 3 | \name{add} 4 | \alias{add} 5 | \alias{add,Simulation,ModelRef-method} 6 | \alias{add,Simulation,DrawsRef-method} 7 | \alias{add,Simulation,OutputRef-method} 8 | \alias{add,Simulation,EvalsRef-method} 9 | \alias{add,Simulation,list-method} 10 | \title{Add a reference to a simulation} 11 | \usage{ 12 | add(sim, ref, ...) 13 | 14 | \S4method{add}{Simulation,ModelRef}(sim, ref, update_saved = TRUE) 15 | 16 | \S4method{add}{Simulation,DrawsRef}(sim, ref, update_saved = TRUE) 17 | 18 | \S4method{add}{Simulation,OutputRef}(sim, ref, update_saved = TRUE) 19 | 20 | \S4method{add}{Simulation,EvalsRef}(sim, ref, update_saved = TRUE) 21 | 22 | \S4method{add}{Simulation,list}(sim, ref, update_saved = TRUE) 23 | } 24 | \arguments{ 25 | \item{sim}{simulation being added to} 26 | 27 | \item{ref}{the reference object being added} 28 | 29 | \item{...}{not used} 30 | 31 | \item{update_saved}{default is TRUE. Determines whether change to simulation 32 | object should be saved to file} 33 | } 34 | \description{ 35 | Adds a ModelRef, DrawsRef, OutputRef, or EvalsRef to a simulation object. 36 | To add a DrawsRef, the corresponding ModelRef must already be added. 37 | Likewise, to add an OutputRef, the corresponding DrawsRef must already be 38 | added. And to add an EvalsRef, the corresponding OutputRef must be added. 39 | One can also pass a list of such objects. 40 | } 41 | \details{ 42 | The modified simulation object is saved to file if \code{update_saved} is 43 | TRUE. 44 | } 45 | -------------------------------------------------------------------------------- /man/add_bold.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/tables.R 3 | \name{add_bold} 4 | \alias{add_bold} 5 | \title{Make a string bold in a certain format} 6 | \usage{ 7 | add_bold(str, output_type) 8 | } 9 | \arguments{ 10 | \item{str}{string or strings (character) to make bold} 11 | 12 | \item{output_type}{output type (see knitr::kable's format)} 13 | } 14 | \description{ 15 | For example, in latex it would take "2" and output "{\\bf 2}"; in html 16 | it would output "2". 17 | } 18 | -------------------------------------------------------------------------------- /man/add_dref_to_list.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/add-to-simulation.R 3 | \name{add_dref_to_list} 4 | \alias{add_dref_to_list} 5 | \title{Internal function to add DrawsRef to a list of DrawsRef objects} 6 | \usage{ 7 | add_dref_to_list(dref, dref_list, sim_dir) 8 | } 9 | \arguments{ 10 | \item{dref}{DrawsRef to add} 11 | 12 | \item{dref_list}{list of DrawsRef objects} 13 | 14 | \item{sim_dir}{sim@dir} 15 | } 16 | \description{ 17 | Makes sure that DrawsRef with this same index is not already in list. 18 | Although not checked, it is assumed that this is only called on a list 19 | of DrawsRef objects all coming from same model. 20 | } 21 | \keyword{internal} 22 | -------------------------------------------------------------------------------- /man/add_eref_to_list.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/add-to-simulation.R 3 | \name{add_eref_to_list} 4 | \alias{add_eref_to_list} 5 | \title{Internal function to add EvalsRef to a list of EvalsRef objects} 6 | \usage{ 7 | add_eref_to_list(eref, eref_list, sim_dir) 8 | } 9 | \arguments{ 10 | \item{eref}{EvalsRef to add} 11 | 12 | \item{eref_list}{list of EvalsRef objects} 13 | 14 | \item{sim_dir}{sim@dir} 15 | } 16 | \description{ 17 | Makes sure that EvalsRef with this same index and method is not already in 18 | list. Although not checked, it is assumed that this is only called on a list 19 | of EvalsRef objects all coming from same model. 20 | } 21 | \keyword{internal} 22 | -------------------------------------------------------------------------------- /man/add_oref_to_list.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/add-to-simulation.R 3 | \name{add_oref_to_list} 4 | \alias{add_oref_to_list} 5 | \title{Internal function to add OutputRef to a list of OutputRef objects} 6 | \usage{ 7 | add_oref_to_list(oref, oref_list, sim_dir) 8 | } 9 | \arguments{ 10 | \item{oref}{OutputRef to add} 11 | 12 | \item{oref_list}{list of OutputRef objects} 13 | 14 | \item{sim_dir}{sim@dir} 15 | } 16 | \description{ 17 | Makes sure that OutputRef with this same index and method is not already in 18 | list. Although not checked, it is assumed that this is only called on a list 19 | of OutputRef objects all coming from same model. 20 | } 21 | \keyword{internal} 22 | -------------------------------------------------------------------------------- /man/aggregate_evals.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/aggregator-class.R 3 | \name{aggregate_evals} 4 | \alias{aggregate_evals} 5 | \title{Apply aggregator to a list of Evals objects} 6 | \usage{ 7 | aggregate_evals(evals_list, aggregator) 8 | } 9 | \arguments{ 10 | \item{evals_list}{a list of Evals objects} 11 | 12 | \item{aggregator}{object of class Aggregator} 13 | } 14 | \description{ 15 | Returns a num_models by num_methods matrix 16 | } 17 | -------------------------------------------------------------------------------- /man/as.data.frame.Evals.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/evals-class.R 3 | \name{as.data.frame.Evals} 4 | \alias{as.data.frame.Evals} 5 | \title{Convert an Evals to a data.frame} 6 | \usage{ 7 | \method{as.data.frame}{Evals}(x, row.names = NULL, optional = FALSE, ...) 8 | } 9 | \arguments{ 10 | \item{x}{object of class \code{\linkS4class{Evals}}} 11 | 12 | \item{row.names}{not used} 13 | 14 | \item{optional}{not used} 15 | 16 | \item{...}{not used} 17 | } 18 | \description{ 19 | This is equivalent to calling \code{as(x, "data.frame")} 20 | } 21 | -------------------------------------------------------------------------------- /man/as.data.frame.Model.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/model-class.R 3 | \name{as.data.frame.Model} 4 | \alias{as.data.frame.Model} 5 | \title{Convert a Model to a data.frame} 6 | \usage{ 7 | \method{as.data.frame}{Model}(x, row.names = NULL, optional = FALSE, ...) 8 | } 9 | \arguments{ 10 | \item{x}{object of class \code{\linkS4class{Model}}} 11 | 12 | \item{row.names}{not used} 13 | 14 | \item{optional}{not used} 15 | 16 | \item{...}{not used} 17 | } 18 | \description{ 19 | Ignores any params that are not length 1 and numeric or character. This is 20 | equivalent to calling \code{as(x, "data.frame")} 21 | } 22 | -------------------------------------------------------------------------------- /man/as.data.frame.listofEvals.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/evals-class.R 3 | \name{as.data.frame.listofEvals} 4 | \alias{as.data.frame.listofEvals} 5 | \title{Convert a list of Evals to a data.frame} 6 | \usage{ 7 | \method{as.data.frame}{listofEvals}(x, row.names = NULL, optional = FALSE, ...) 8 | } 9 | \arguments{ 10 | \item{x}{a listofEvals object} 11 | 12 | \item{row.names}{not used} 13 | 14 | \item{optional}{not used} 15 | 16 | \item{...}{not used} 17 | } 18 | \description{ 19 | When \code{\link{load}} generates a list of Evals, it assigns this 20 | to be of (S3) class listofEvals, inherited from list, so that this function 21 | will be invoked instead of as.data.frame.list, which is defined in base. 22 | } 23 | -------------------------------------------------------------------------------- /man/as.data.frame.listofModels.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/model-class.R 3 | \name{as.data.frame.listofModels} 4 | \alias{as.data.frame.listofModels} 5 | \title{Convert a List of Models to a data.frame} 6 | \usage{ 7 | \method{as.data.frame}{listofModels}(x, row.names = NULL, optional = FALSE, ...) 8 | } 9 | \arguments{ 10 | \item{x}{list} 11 | 12 | \item{row.names}{not used} 13 | 14 | \item{optional}{not used} 15 | 16 | \item{...}{not used} 17 | } 18 | \description{ 19 | When \code{\link{load}} generates a list of Models, it assigns this 20 | to be of (S3) class listofModels, inherited from list, so that this function 21 | will be invoked instead of as.data.frame.list, which is defined in base. 22 | } 23 | -------------------------------------------------------------------------------- /man/cash-Model-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/model-class.R 3 | \name{$,Model-method} 4 | \alias{$,Model-method} 5 | \title{Get element of \code{\linkS4class{Model}}'s \code{params} list} 6 | \usage{ 7 | \S4method{$}{Model}(x, name) 8 | } 9 | \arguments{ 10 | \item{x}{object of class \code{\linkS4class{Model}}} 11 | 12 | \item{name}{name of an element appearing in \code{x@params}} 13 | } 14 | \description{ 15 | Get element of \code{\linkS4class{Model}}'s \code{params} list 16 | } 17 | -------------------------------------------------------------------------------- /man/catsim.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{catsim} 4 | \alias{catsim} 5 | \title{Concatenate and print for the simulator} 6 | \usage{ 7 | catsim(...) 8 | } 9 | \arguments{ 10 | \item{...}{arguments to be passed to \code{\link{cat}}} 11 | } 12 | \description{ 13 | For internal use. This calls \code{\link{cat}} only when 14 | \code{getOption("simulator.verbose")}. 15 | } 16 | -------------------------------------------------------------------------------- /man/create.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/create.R 3 | \name{create} 4 | \alias{create} 5 | \title{Create template for a new set of simulations} 6 | \usage{ 7 | create(dir = "./my_sims") 8 | } 9 | \arguments{ 10 | \item{dir}{where to create the skeleton of a new set of simulations} 11 | } 12 | \description{ 13 | This function is the fastest way to get started. Creates the skeleton of a 14 | simulation. 15 | } 16 | \examples{ 17 | \dontrun{ 18 | create("./examples") 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /man/describe.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/manage.R 3 | \name{describe} 4 | \alias{describe} 5 | \title{Describe the contents of a simulator directory} 6 | \usage{ 7 | describe(dir = ".") 8 | } 9 | \arguments{ 10 | \item{dir}{name of the directory where directory named "files" exists} 11 | } 12 | \description{ 13 | Describe the contents of a simulator directory 14 | } 15 | -------------------------------------------------------------------------------- /man/do_in_parallel.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/parallel.R 3 | \name{do_in_parallel} 4 | \alias{do_in_parallel} 5 | \title{Do a function in parallel.} 6 | \usage{ 7 | do_in_parallel( 8 | function_to_do, 9 | function_params, 10 | save_to_file, 11 | save_params, 12 | socket_names, 13 | libraries, 14 | save_locally = TRUE 15 | ) 16 | } 17 | \arguments{ 18 | \item{function_to_do}{this is the function that will be done in parallel} 19 | 20 | \item{function_params}{a list where \code{function_params[[i]]} is 21 | a named list of parameters to be passed to \code{function_to_do} for 22 | job \code{i}} 23 | 24 | \item{save_to_file}{function that saves stuff to file} 25 | 26 | \item{save_params}{a list where \code{save_params[[i]]} 27 | is a named list of parameters to be passed to \code{save_to_file} for 28 | job \code{i}. Each \code{save_params[[i]]} must include \code{out_dir}, 29 | which is location where file is to be saved.} 30 | 31 | \item{socket_names}{quoting from \code{\link[parallel]{makePSOCKcluster}}: 32 | "either a character vector of host names on which to run the worker 33 | copies of R, or a positive integer (in which case that number of 34 | copies is run on localhost)."} 35 | 36 | \item{libraries}{character vector of R packages that will be needed on the 37 | slaves.} 38 | 39 | \item{save_locally}{if TRUE, then files will be saved on slaves. If FALSE, 40 | they will be saved on master.} 41 | } 42 | \description{ 43 | This is an internal function. The function_to_do is done in parallel 44 | and its output is saved either on slave (locally) or on master. 45 | } 46 | \keyword{internal} 47 | -------------------------------------------------------------------------------- /man/draws.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get-from-simulation.R 3 | \name{draws} 4 | \alias{draws} 5 | \title{Get one or more draws from a simulation} 6 | \usage{ 7 | draws(sim, ..., subset = NULL, index, reference = FALSE) 8 | } 9 | \arguments{ 10 | \item{sim}{a simulation object} 11 | 12 | \item{...}{logical conditions to specify a subset of models. Conditions can 13 | only involve params of model that have length 1 and are of class 14 | numeric or character.} 15 | 16 | \item{subset}{a vector of integers indexing the models or a vector of model 17 | names. To select models based on parameter values, use \code{...}. 18 | However, using \code{...} is slower than using subset.} 19 | 20 | \item{index}{a vector of positive integers specifying which draws objects 21 | are desired. If missing, then all draws' outputs are returned.} 22 | 23 | \item{reference}{whether to return the ModelRef or the Model object itself} 24 | } 25 | \description{ 26 | Returns either the draws objects themselves or references to them. See 27 | \code{\link{model}} function for more information on the \code{...} and 28 | \code{subset} arguments, which are used to specify a subset of the models. 29 | } 30 | \examples{ 31 | \dontrun{ 32 | # suppose previously we had run the following: 33 | sim <- new_simulation(name = "normal-example", 34 | label = "Normal Mean Estimation", 35 | dir = tempdir()) \%>\% 36 | generate_model(make_my_example_model, n = 20) \%>\% 37 | simulate_from_model(nsim = 50, index = 1:3) 38 | # then we could get the simulated draws as follows: 39 | d <- draws(sim) 40 | d@draws$r1.1 # first random draw 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /man/evals.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get-from-simulation.R 3 | \name{evals} 4 | \alias{evals} 5 | \title{Get one or more evals from a simulation} 6 | \usage{ 7 | evals(sim, ..., subset = NULL, index, methods, reference = FALSE) 8 | } 9 | \arguments{ 10 | \item{sim}{a simulation object} 11 | 12 | \item{...}{logical conditions to specify a subset of models. Conditions can 13 | only involve params of model that have length 1 and are of class 14 | numeric or character.} 15 | 16 | \item{subset}{a vector of integers indexing the models or a vector of model 17 | names. To select models based on parameter values, use \code{...}. 18 | However, using \code{...} is slower than using subset.} 19 | 20 | \item{index}{a vector of positive integers specifying which draws' objects 21 | are desired. If missing, then all draws' evals are returned.} 22 | 23 | \item{methods}{character vector of method names of interest. If missing, 24 | then all methods' evals are returned} 25 | 26 | \item{reference}{whether to return the ModelRef or the Model object itself} 27 | } 28 | \description{ 29 | Returns either the Evals object itself or a reference to it. 30 | } 31 | \examples{ 32 | \dontrun{ 33 | # suppose previously we had run the following: 34 | sim <- new_simulation(name = "normal-example", 35 | label = "Normal Mean Estimation", 36 | dir = tempdir()) \%>\% 37 | generate_model(make_my_example_model, n = 20) \%>\% 38 | simulate_from_model(nsim = 50, index = 1:3) \%>\% 39 | run_method(my_example_method) \%>\% 40 | evaluate(my_example_loss) 41 | # then we could get the metric evaluated on the method's output: 42 | e <- evals(sim) 43 | # we can export it as a data.frame 44 | as.data.frame(e) 45 | # or we can get at a particular draw-method-metric triplet 46 | e@evals$`my-method`$r1.1$myloss 47 | } 48 | } 49 | \seealso{ 50 | \code{\link{as.data.frame}} 51 | } 52 | -------------------------------------------------------------------------------- /man/evaluate.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/evaluate.R 3 | \name{evaluate} 4 | \alias{evaluate} 5 | \title{Evaluate outputs of methods according to provided metrics.} 6 | \usage{ 7 | evaluate(object, metrics) 8 | } 9 | \arguments{ 10 | \item{object}{object of class \code{\linkS4class{OutputRef}} as produced by 11 | \code{\link{run_method}} (or list of such objects). If 12 | \code{object} is a \code{\linkS4class{Simulation}}, then function is applied 13 | to the referenced outputs in that simulation and returns the same 14 | \code{Simulation} object but with references added to the new evals 15 | created.} 16 | 17 | \item{metrics}{a list of \code{\linkS4class{Metric}} objects or a single 18 | \code{\linkS4class{Metric}} object.} 19 | } 20 | \description{ 21 | Given a \code{\linkS4class{Metric}} object or list of \code{\linkS4class{Metric}} objects, 22 | this function evaluates an \code{\linkS4class{Output}} object according to these 23 | metrics. The computed values of the metrics are saved to file. The "user" 24 | time to run the method (as measured by \code{\link{system.time}}) is added 25 | to \code{metrics} by default unless one of the passed metrics has name 26 | "time". 27 | } 28 | \details{ 29 | This function creates objects of class \code{\linkS4class{Evals}} and saves each to 30 | file (at dir/model_name//r__evals.Rdata. Since 31 | evaluating metrics is usually (in statistical methodological papers) fast, 32 | parallel functionality has not been developed for the evaluation component. 33 | } 34 | \examples{ 35 | \dontrun{ 36 | # suppose previously we had run the following: 37 | sim <- new_simulation(name = "normal-example", 38 | label = "Normal Mean Estimation", 39 | dir = tempdir()) \%>\% 40 | generate_model(make_my_example_model, n = 20) \%>\% 41 | simulate_from_model(nsim = 50, index = 1:3) \%>\% 42 | run_method(my_example_method) 43 | # then we could add 44 | sim <- evaluate(sim, my_example_loss) 45 | } 46 | } 47 | \seealso{ 48 | \code{\link{generate_model}} \code{\link{simulate_from_model}} 49 | \code{\link{run_method}} 50 | } 51 | -------------------------------------------------------------------------------- /man/evaluate_internal.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/evaluate.R 3 | \name{evaluate_internal} 4 | \alias{evaluate_internal} 5 | \title{Evaluate outputs of methods according to provided metrics.} 6 | \usage{ 7 | evaluate_internal( 8 | metrics, 9 | dir = ".", 10 | model_name, 11 | index, 12 | method_names, 13 | out_loc = "out" 14 | ) 15 | } 16 | \arguments{ 17 | \item{metrics}{a list of \code{\linkS4class{Metric}} objects or a single 18 | \code{\linkS4class{Metric}} object} 19 | 20 | \item{dir}{the directory where \code{\linkS4class{Model}} object was saved (by 21 | \code{\link{generate_model}})} 22 | 23 | \item{model_name}{the \code{\linkS4class{Model}} object's \code{name} attribute} 24 | 25 | \item{index}{the index of a computed \code{\linkS4class{Draws}} object. Can 26 | alternately be a vector of such indices.} 27 | 28 | \item{method_names}{the \code{\linkS4class{Method}} objects' \code{name} attributes 29 | as a character vector.} 30 | 31 | \item{out_loc}{(optional) a length-1 character vector that gives location 32 | (relative to model's path) that method outputs are stored.} 33 | } 34 | \description{ 35 | Given a \code{\linkS4class{Metric}} object or list of \code{\linkS4class{Metric}} objects, 36 | this function evaluates an \code{\linkS4class{Output}} object according to these 37 | metrics. The computed values of the metrics are saved to file. 38 | } 39 | \details{ 40 | This function creates objects of class \code{\linkS4class{Evals}} and saves each to 41 | file (at dir/model_name//r__evals.Rdata. Since 42 | evaluating metrics is usually (in statistical methodological papers) fast, 43 | parallel functionality has not been developed for the evaluation component. 44 | } 45 | -------------------------------------------------------------------------------- /man/evaluate_single.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/evaluate.R 3 | \name{evaluate_single} 4 | \alias{evaluate_single} 5 | \title{Run one or more metrics on outputs.} 6 | \usage{ 7 | evaluate_single(metrics, model, output, draws = NULL) 8 | } 9 | \arguments{ 10 | \item{metrics}{a list of \code{\linkS4class{Metric}} objects} 11 | 12 | \item{model}{a \code{\linkS4class{Model}} object} 13 | 14 | \item{output}{a \code{\linkS4class{Output}} object} 15 | 16 | \item{draws}{(optional) a \code{\linkS4class{Draws}} object or NULL} 17 | } 18 | \description{ 19 | This is an internal function. Users should call the wrapper function 20 | \code{\link{evaluate}}. Here "single" refers to a single output (and 21 | thus a single method, though not necessarily a single index). 22 | The metrics provided are run and saved together in a file. 23 | } 24 | -------------------------------------------------------------------------------- /man/figures/simulator-hex3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobbien/simulator/3d325ec78d48c57cbdc16d1455c2ffcc85ae8bb1/man/figures/simulator-hex3.png -------------------------------------------------------------------------------- /man/generate_model.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/models.R 3 | \name{generate_model} 4 | \alias{generate_model} 5 | \title{Generate a model.} 6 | \usage{ 7 | generate_model(object = ".", make_model, ..., seed = 123, vary_along = NULL) 8 | } 9 | \arguments{ 10 | \item{object}{the name of the directory where directory named "files" exists 11 | (or should be created) to save \code{\linkS4class{Model}} object in. 12 | Default is current working directory. Or can be an object of class 13 | \code{\linkS4class{Simulation}}, in which case the \code{object@dir} is used 14 | and a simulation object is returned instead of an object of class 15 | \code{\linkS4class{ModelRef}}.} 16 | 17 | \item{make_model}{a function that outputs an object of class 18 | \code{\linkS4class{Model}}. Or a list of such functions.} 19 | 20 | \item{...}{optional parameters that may be passed to make_model} 21 | 22 | \item{seed}{an integer seed for the random number generator.} 23 | 24 | \item{vary_along}{character vector with all elements contained in names(...) 25 | See description for more details.} 26 | } 27 | \description{ 28 | This function executes the \code{make_model} function provided by the user 29 | and writes to file the resulting \code{\linkS4class{Model}} object(s). For example, 30 | when simulating regression with a fixed design, \code{X} would be generated 31 | in this function and \code{n}, \code{p}, \code{beta}, and \code{sigma} would 32 | also be specified. 33 | } 34 | \details{ 35 | When \code{make_model} has arguments, these can be passed using \code{...}. 36 | These will be passed directly to \code{make_model} except for any arguments 37 | named in \code{vary_along}. These arguments should be lists and a separate 38 | model will be created for each combination of elements in these lists. For 39 | example, if \code{vary_along = c("n", "p")}, then we can pass 40 | \code{n=as.list(c(50, 100, 150))} and \code{p=as.list(c(10, 100))} and 6 41 | models will be created, one for each pair of \code{n} and \code{p}. For each 42 | pair (n,p), a distinct extension is added to the end of the model name. This 43 | extension is generated using a hash function so that different values of the 44 | vary_along parameters will lead to different model name extensions. This 45 | ensures that if one later decides to add more values of the vary_along 46 | parameters, this will not lead to pre-existing files being overwritten 47 | (unless the same values of the vary_along combination are used again. 48 | 49 | If \code{object} is a directory name, the function returns a reference or 50 | list of references to the model(s) generated. If \code{object} is a 51 | \code{Simulation}, then function returns the same \code{Simulation} object 52 | but with references added to the new models created. These changes to the 53 | \code{Simulation} object are saved to file. 54 | 55 | \code{make_model} is called generating an object of class 56 | \code{\linkS4class{Model}}, called \code{model}, which is saved to 57 | \code{dir/name/model.Rdata} (where \code{name} is the name attribute of 58 | \code{model}). This file also contains the random number generator state and 59 | other information such as the function \code{make_model} itself and the date 60 | when \code{model} was created. 61 | } 62 | \examples{ 63 | # initialize a new simulation 64 | sim <- new_simulation(name = "normal-example", 65 | label = "Normal Mean Estimation", 66 | dir = tempdir()) 67 | # generate a model (and add it to the simulation) 68 | sim <- generate_model(sim, make_my_example_model, n = 20) 69 | # generate a sequence of models (and add them to the simulation) 70 | sim <- generate_model(sim, make_my_example_model, 71 | n = list(10, 20, 30), 72 | vary_along = "n") 73 | } 74 | \seealso{ 75 | \code{\link{new_model}} \code{\link{simulate_from_model}} 76 | \code{\link{run_method}} 77 | } 78 | -------------------------------------------------------------------------------- /man/get_contents.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/manage.R 3 | \name{get_contents} 4 | \alias{get_contents} 5 | \title{Get the contents of a simulator directory} 6 | \usage{ 7 | get_contents(dir = ".", out_loc = "out") 8 | } 9 | \arguments{ 10 | \item{dir}{name of the directory where directory named "files" exists} 11 | 12 | \item{out_loc}{a length-1 character vector that gives location 13 | (relative to model's path) that method outputs are stored.This can be 14 | useful for staying organized when multiple simulations are based on 15 | the same Model and Draws objects. Usually this is just "out"} 16 | } 17 | \description{ 18 | This function gives detailed information about what is being stored in the 19 | "files" directory. In particular, it gives the complete paths for all the 20 | draws, outputs, and evals files. This can be useful in situations in which 21 | the draws or outputs files are no longer needed and take up a lot of memory. 22 | In such a case a user could delete these files with a command such as 23 | \code{system(paste(c("rm", contents$out_files), collapse = " "))}. 24 | That said, one must be cautious in deleting these files since the 25 | simulator generally assumes that earlier stages' files will be available 26 | and so deleting these may cause errors. However, if one is essentially 27 | finished with a simulation and evaluated metrics have been computed and if 28 | the methods' raw outputs are taking up a lot of disk space, then one might 29 | consider deleting the out_files (and/or the draws_files). 30 | } 31 | -------------------------------------------------------------------------------- /man/get_files_not_in_simulations.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/manage.R 3 | \name{get_files_not_in_simulations} 4 | \alias{get_files_not_in_simulations} 5 | \title{Find files in simulator directory not referred to by any simulations} 6 | \usage{ 7 | get_files_not_in_simulations(dir, out_loc = "out") 8 | } 9 | \arguments{ 10 | \item{dir}{name of the directory where directory named "files" exists} 11 | 12 | \item{out_loc}{a length-1 character vector that gives location 13 | (relative to model's path) that method outputs are stored.This can be 14 | useful for staying organized when multiple simulations are based on 15 | the same Model and Draws objects. Usually this is just "out"} 16 | } 17 | \description{ 18 | Once one has completed all simulation studies, this function can be called 19 | to identify any files that may have been created along the way that are no 20 | longer being used in any simulations. It would then be safe to delete these 21 | files. 22 | } 23 | -------------------------------------------------------------------------------- /man/get_model_indices.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get-from-simulation.R 3 | \name{get_model_indices} 4 | \alias{get_model_indices} 5 | \title{Returns indices of a specified subset of sim@model_refs} 6 | \usage{ 7 | get_model_indices(sim, subset) 8 | } 9 | \arguments{ 10 | \item{sim}{a simulation object} 11 | 12 | \item{subset}{a vector indicating which models should be returned.} 13 | } 14 | \description{ 15 | See \code{\link{model}} for information about the various formats of subset. 16 | } 17 | -------------------------------------------------------------------------------- /man/get_relative_path.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{get_relative_path} 4 | \alias{get_relative_path} 5 | \title{Get relative path} 6 | \usage{ 7 | get_relative_path(base_path, path) 8 | } 9 | \arguments{ 10 | \item{base_path}{the base path} 11 | 12 | \item{path}{a specific path} 13 | } 14 | \description{ 15 | Given a base path and a specific path, returns a string str such that 16 | file.path(base_path, str) is the same location as path. 17 | } 18 | -------------------------------------------------------------------------------- /man/get_simulation_with_all_files.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/manage.R 3 | \name{get_simulation_with_all_files} 4 | \alias{get_simulation_with_all_files} 5 | \title{Returns a simulation object containing references to all files in directory} 6 | \usage{ 7 | get_simulation_with_all_files(dir, out_loc = "out") 8 | } 9 | \arguments{ 10 | \item{dir}{name of the directory where directory named "files" exists} 11 | 12 | \item{out_loc}{a length-1 character vector that gives location 13 | (relative to model's path) that method outputs are stored.This can be 14 | useful for staying organized when multiple simulations are based on 15 | the same Model and Draws objects. Usually this is just "out"} 16 | } 17 | \description{ 18 | Returns a simulation object containing references to all files in directory 19 | } 20 | -------------------------------------------------------------------------------- /man/ggplot_eval.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/plot_eval.R 3 | \name{ggplot_eval} 4 | \alias{ggplot_eval} 5 | \title{Make a boxplot of a metric for each method using ggplot2} 6 | \usage{ 7 | ggplot_eval( 8 | evals_df, 9 | metric_name, 10 | method_name, 11 | method_label, 12 | main, 13 | facet_mains, 14 | ylab, 15 | ylim, 16 | nrow, 17 | ncol 18 | ) 19 | } 20 | \description{ 21 | Make a boxplot of a metric for each method using ggplot2 22 | } 23 | \keyword{internal} 24 | -------------------------------------------------------------------------------- /man/grapes-greater-than-grapes.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/import_from.R 3 | \name{\%>\%} 4 | \alias{\%>\%} 5 | \title{This is not magrittr's pipe (actually it is)} 6 | \description{ 7 | This is not magrittr's pipe (actually it is) 8 | } 9 | \keyword{internal} 10 | -------------------------------------------------------------------------------- /man/load-DrawsRef-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/load.R 3 | \name{load,DrawsRef-method} 4 | \alias{load,DrawsRef-method} 5 | \title{Load a DrawsRef} 6 | \usage{ 7 | \S4method{load}{DrawsRef}(file) 8 | } 9 | \arguments{ 10 | \item{file}{object to load} 11 | } 12 | \description{ 13 | Load a DrawsRef 14 | } 15 | -------------------------------------------------------------------------------- /man/load-EvalsRef-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/load.R 3 | \name{load,EvalsRef-method} 4 | \alias{load,EvalsRef-method} 5 | \title{Load an EvalsRef} 6 | \usage{ 7 | \S4method{load}{EvalsRef}(file) 8 | } 9 | \arguments{ 10 | \item{file}{object to load} 11 | } 12 | \description{ 13 | Load an EvalsRef 14 | } 15 | -------------------------------------------------------------------------------- /man/load-ModelRef-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/load.R 3 | \name{load,ModelRef-method} 4 | \alias{load,ModelRef-method} 5 | \title{Load a ModelRef} 6 | \usage{ 7 | \S4method{load}{ModelRef}(file) 8 | } 9 | \arguments{ 10 | \item{file}{object to load} 11 | } 12 | \description{ 13 | Load a ModelRef 14 | } 15 | -------------------------------------------------------------------------------- /man/load-OutputRef-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/load.R 3 | \name{load,OutputRef-method} 4 | \alias{load,OutputRef-method} 5 | \title{Load an OutputRef} 6 | \usage{ 7 | \S4method{load}{OutputRef}(file) 8 | } 9 | \arguments{ 10 | \item{file}{object to load} 11 | } 12 | \description{ 13 | Load an OutputRef 14 | } 15 | -------------------------------------------------------------------------------- /man/load-list-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/load.R 3 | \name{load,list-method} 4 | \alias{load,list-method} 5 | \title{Load a list of reference objects} 6 | \usage{ 7 | \S4method{load}{list}(file) 8 | } 9 | \arguments{ 10 | \item{file}{list of objects to load} 11 | } 12 | \description{ 13 | Load a list of reference objects 14 | } 15 | -------------------------------------------------------------------------------- /man/load_draws.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/draws.R 3 | \name{load_draws} 4 | \alias{load_draws} 5 | \title{Load one or more draws objects from file.} 6 | \usage{ 7 | load_draws(dir, model_name, index, more_info = FALSE, simulator.files = NULL) 8 | } 9 | \arguments{ 10 | \item{dir}{the directory passed to \code{\link{generate_model}})} 11 | 12 | \item{model_name}{the Model object's \code{name} attribute} 13 | 14 | \item{index}{a vector of positive integers.} 15 | 16 | \item{more_info}{if TRUE, then returns additional information such as 17 | state of RNG after calling \code{\link{generate_model}}} 18 | 19 | \item{simulator.files}{if NULL, then \code{getOption("simulator.files")} 20 | will be used.} 21 | } 22 | \description{ 23 | After \code{\link{simulate_from_model}} has been called, this function can 24 | be used to load one or more of the saved \code{\linkS4class{Draws}} object(s) 25 | (along with RNG information). If multiple indices are provided, these will 26 | be combined into a new single \code{\linkS4class{Draws}} object. 27 | If simulation object is available, it is easier to use the function 28 | \code{\link{draws}} to load it. 29 | } 30 | \seealso{ 31 | \code{\link{simulate_from_model}} \code{\link{draws}} 32 | } 33 | -------------------------------------------------------------------------------- /man/load_evals.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/evaluate.R 3 | \name{load_evals} 4 | \alias{load_evals} 5 | \alias{load_evals_from_ref} 6 | \title{Load one or more Evals objects from file.} 7 | \usage{ 8 | load_evals( 9 | dir, 10 | model_name, 11 | index, 12 | method_names, 13 | metric_names = NULL, 14 | out_loc = "out", 15 | simulator.files = NULL 16 | ) 17 | 18 | load_evals_from_ref(ref, metric_names = NULL) 19 | } 20 | \arguments{ 21 | \item{dir}{the directory passed to \code{\link{generate_model}})} 22 | 23 | \item{model_name}{the \code{\linkS4class{Model}} object's \code{name}} 24 | 25 | \item{index}{a vector of positive integers.} 26 | 27 | \item{method_names}{the \code{name} of one or more \code{\linkS4class{Method}} 28 | objects.} 29 | 30 | \item{metric_names}{(optional) a character vector of which elements of 31 | evals should be loaded. If NULL, then all elements are loaded.} 32 | 33 | \item{out_loc}{only needed if it was used in call to} 34 | 35 | \item{simulator.files}{if NULL, then \code{getOption("simulator.files")} 36 | will be used. 37 | \code{\link{run_method}}.} 38 | 39 | \item{ref}{an object of class \code{\linkS4class{EvalsRef}}} 40 | } 41 | \description{ 42 | After \code{\link{evaluate}} has been called, this function can 43 | be used to load one or more of the saved \code{\linkS4class{Evals}} object(s). 44 | If multiple indices are provided, these will be combined by index into a 45 | new single \code{\linkS4class{Evals}} object. If multiple methods are provided, 46 | a list of \code{\linkS4class{Evals}} objects will be returned. 47 | } 48 | \seealso{ 49 | \code{\link{load_model}} \code{\link{load_draws}} 50 | \code{\link{as.data.frame.Evals}} 51 | } 52 | -------------------------------------------------------------------------------- /man/load_libraries_on_cluster.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/parallel.R 3 | \name{load_libraries_on_cluster} 4 | \alias{load_libraries_on_cluster} 5 | \title{Load necessary libraries on a cluster} 6 | \usage{ 7 | load_libraries_on_cluster(cl, libs) 8 | } 9 | \description{ 10 | Load necessary libraries on a cluster 11 | } 12 | \keyword{internal} 13 | -------------------------------------------------------------------------------- /man/load_model.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/models.R 3 | \name{load_model} 4 | \alias{load_model} 5 | \title{Load a model from file.} 6 | \usage{ 7 | load_model(dir, model_name, more_info = FALSE, simulator.files = NULL) 8 | } 9 | \arguments{ 10 | \item{dir}{the directory passed to \code{\link{generate_model}})} 11 | 12 | \item{model_name}{the Model object's \code{name} attribute} 13 | 14 | \item{more_info}{if TRUE, then returns additional information such as 15 | state of RNG after calling \code{\link{generate_model}}} 16 | 17 | \item{simulator.files}{if NULL, then \code{getOption("simulator.files")} 18 | will be used.} 19 | } 20 | \description{ 21 | After \code{\link{generate_model}} has been called, this function can be used 22 | to load the saved \code{\linkS4class{Model}} object (along with the RNG state and 23 | other information if desired). 24 | } 25 | \details{ 26 | Depending on \code{more_info}, either returns \code{\linkS4class{Model}} object 27 | or a list containing \code{\linkS4class{Model}} object and other information. 28 | If simulation object is available, it is easier to use the function 29 | \code{\link{model}} to load the model. 30 | } 31 | \seealso{ 32 | \code{\link{generate_model}} \code{\link{model}} 33 | } 34 | -------------------------------------------------------------------------------- /man/load_outputs.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/methods.R 3 | \name{load_outputs} 4 | \alias{load_outputs} 5 | \alias{load_outputs_from_ref} 6 | \title{Load one or more output objects from file.} 7 | \usage{ 8 | load_outputs( 9 | dir, 10 | model_name, 11 | index, 12 | method_name, 13 | out_names = NULL, 14 | out_loc = "out", 15 | more_info = FALSE, 16 | simulator.files = NULL 17 | ) 18 | 19 | load_outputs_from_ref(ref, out_names = NULL) 20 | } 21 | \arguments{ 22 | \item{dir}{the directory passed to \code{\link{generate_model}})} 23 | 24 | \item{model_name}{the \code{\linkS4class{Model}} object's \code{name}} 25 | 26 | \item{index}{a vector of positive integers.} 27 | 28 | \item{method_name}{the \code{\linkS4class{Method}} object's \code{name}} 29 | 30 | \item{out_names}{a character vector of which elements of output should be 31 | loaded. If NULL, then all elements are loaded.} 32 | 33 | \item{out_loc}{only needed if it was used in call to 34 | \code{\link{run_method}}.} 35 | 36 | \item{more_info}{if TRUE, then returns additional information such as 37 | state of RNG after calling \code{\link{simulate_from_model}}} 38 | 39 | \item{simulator.files}{if NULL, then \code{getOption("simulator.files")} 40 | will be used.} 41 | 42 | \item{ref}{an object of class \code{\linkS4class{OutputRef}}} 43 | } 44 | \description{ 45 | After \code{\link{run_method}} has been called, this function can 46 | be used to load one or more of the saved \code{\linkS4class{Output}} object(s). 47 | If multiple indices are provided, these will be combined 48 | into a new single \code{\linkS4class{Output}} object. 49 | If simulation object is available, it is easier to use the function 50 | \code{\link{output}} to load it. 51 | } 52 | \seealso{ 53 | \code{\link{run_method}} \code{\link{output}} 54 | } 55 | \keyword{internal} 56 | -------------------------------------------------------------------------------- /man/load_simulation.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/simulation-class.R 3 | \name{load_simulation} 4 | \alias{load_simulation} 5 | \title{Load a simulation object} 6 | \usage{ 7 | load_simulation(name, dir = ".") 8 | } 9 | \arguments{ 10 | \item{name}{a short name identifier. Must be alphanumeric.} 11 | 12 | \item{dir}{directory that contains "files" directory for this simulation} 13 | } 14 | \description{ 15 | Loads an object of class \code{\linkS4class{Simulation}}. Note that \code{dir} 16 | gives the directory where the Simulation object is stored. Thus, if the 17 | working directory is different from the working directory when the Simulation 18 | object was created, then \code{dir} will be different from the one passed to 19 | \code{\link{new_simulation}}. 20 | } 21 | \examples{ 22 | sim <- new_simulation(name = "normal-example", 23 | label = "Normal Mean Estimation", 24 | dir = tempdir()) 25 | rm(sim) 26 | sim <- load_simulation("normal-example", dir = tempdir()) 27 | } 28 | \seealso{ 29 | \code{\link{new_simulation}} \code{\link{save_simulation}} 30 | } 31 | -------------------------------------------------------------------------------- /man/make_my_example_model.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/examples.R 3 | \name{make_my_example_model} 4 | \alias{make_my_example_model} 5 | \title{Make My Example Model} 6 | \usage{ 7 | make_my_example_model(n) 8 | } 9 | \arguments{ 10 | \item{n}{number of i.i.d. draws} 11 | } 12 | \description{ 13 | This function is used in the examples. It returns a 14 | \code{\linkS4class{Model}} object. In particular, it represents 15 | \code{n} i.i.d. draws from a normal with mean 2 and variance 1. 16 | } 17 | \seealso{ 18 | \code{\link{my_example_method}} \code{\link{my_example_loss}} 19 | } 20 | -------------------------------------------------------------------------------- /man/memory_as_string.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{memory_as_string} 4 | \alias{memory_as_string} 5 | \title{Write memory in human readable way} 6 | \usage{ 7 | memory_as_string(memory_in_bytes) 8 | } 9 | \arguments{ 10 | \item{memory_in_bytes}{the amount of memory in Bytes.} 11 | } 12 | \description{ 13 | Write memory in human readable way 14 | } 15 | -------------------------------------------------------------------------------- /man/model.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get-from-simulation.R 3 | \name{model} 4 | \alias{model} 5 | \title{Get one or more models from a simulation} 6 | \usage{ 7 | model(sim, ..., subset = NULL, reference = FALSE) 8 | } 9 | \arguments{ 10 | \item{sim}{a simulation object} 11 | 12 | \item{...}{logical conditions to specify a subset of models. Conditions can 13 | only involve params of model that have length 1 and are of class 14 | numeric or character.} 15 | 16 | \item{subset}{a vector of integers indexing the models or a vector of model 17 | names. To select models based on parameter values, use \code{...}. 18 | However, using \code{...} is slower than using subset.} 19 | 20 | \item{reference}{whether to return the ModelRef or the Model object itself} 21 | } 22 | \description{ 23 | Returns either the models themselves or references to them. 24 | } 25 | \details{ 26 | There are two main ways to specify a subset of the models. (1) The easiest 27 | way is by writing a conditional expression involving the parameters and 28 | passing it through \code{...}. For example, \code{n > 100 & p <= 20}. 29 | Only parameters that are length one and either numeric or character can be 30 | used in these expressions. (2) The faster way to retrieve a subset of 31 | models is to use the \code{subset} argument. This can be either a set of 32 | numerical values (specifying which models to load based on the order in 33 | which the models are stored in the simulation object. This order can be 34 | ascertained by printing the simulation object.) or as a set of a character 35 | vector of the model names desired. 36 | 37 | While approach (1) is very convenient, it requires loading all models from 38 | file. This may be slow in situations in which there are a lot of models 39 | and/or the models are large and thus slow to load. 40 | } 41 | -------------------------------------------------------------------------------- /man/model_names.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/simulation-class.R 3 | \name{model_names} 4 | \alias{model_names} 5 | \title{Get model names in a Simulation} 6 | \usage{ 7 | model_names(sim) 8 | } 9 | \arguments{ 10 | \item{sim}{object of class \code{\linkS4class{Simulation}}} 11 | } 12 | \description{ 13 | Get model names in a Simulation 14 | } 15 | -------------------------------------------------------------------------------- /man/models_as_data.frame.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/model-class.R 3 | \name{models_as_data.frame} 4 | \alias{models_as_data.frame} 5 | \title{Convert a list of Model objects into a data.frame} 6 | \usage{ 7 | models_as_data.frame(m) 8 | } 9 | \arguments{ 10 | \item{m}{model object} 11 | } 12 | \description{ 13 | Ignores any params that are not length 1 and numeric or character 14 | } 15 | -------------------------------------------------------------------------------- /man/my_example_loss.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/examples.R 3 | \docType{data} 4 | \name{my_example_loss} 5 | \alias{my_example_loss} 6 | \title{My Example Loss} 7 | \format{ 8 | An object of class \code{Metric} of length 1. 9 | } 10 | \usage{ 11 | my_example_loss 12 | } 13 | \description{ 14 | This \code{\linkS4class{Metric}} object is used in the examples. It is 15 | squared error loss. 16 | } 17 | \seealso{ 18 | \code{\link{make_my_example_model}} \code{\link{my_example_loss}} 19 | } 20 | \keyword{datasets} 21 | -------------------------------------------------------------------------------- /man/my_example_method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/examples.R 3 | \docType{data} 4 | \name{my_example_method} 5 | \alias{my_example_method} 6 | \title{My Example Method} 7 | \format{ 8 | An object of class \code{Method} of length 1. 9 | } 10 | \usage{ 11 | my_example_method 12 | } 13 | \description{ 14 | This \code{\linkS4class{Method}} object is used in the examples. It is 15 | the sample mean of the data. 16 | } 17 | \seealso{ 18 | \code{\link{make_my_example_model}} \code{\link{my_example_loss}} 19 | } 20 | \keyword{datasets} 21 | -------------------------------------------------------------------------------- /man/new_aggregator.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/aggregator-class.R 3 | \name{new_aggregator} 4 | \alias{new_aggregator} 5 | \title{Create an Aggregator object} 6 | \usage{ 7 | new_aggregator(label, aggregate) 8 | } 9 | \arguments{ 10 | \item{label}{a human readable label} 11 | 12 | \item{aggregate}{a function with argument \code{ev} that is a list of length 13 | equal to the number of draws with each element itself being a named list. 14 | Each element of this list corresponds to a metric that has been computed. 15 | In particular, given an \code{\linkS4class{Evals}} object \code{o}, 16 | \code{aggregate} takes as input \code{o@evals[[method_name]]} (which is a 17 | list of the kind just described). The function aggregate should return a 18 | scalar.} 19 | } 20 | \description{ 21 | Creates a new \code{\linkS4class{Aggregator}} object. 22 | } 23 | -------------------------------------------------------------------------------- /man/new_extended_method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/extended-method-class.R 3 | \name{new_extended_method} 4 | \alias{new_extended_method} 5 | \title{Create an ExtendedMethod object} 6 | \usage{ 7 | new_extended_method(name, label, base_method, extended_method) 8 | } 9 | \arguments{ 10 | \item{name}{a short name identifier. Must be alphanumeric.} 11 | 12 | \item{label}{a longer, human readable label that can have other characters 13 | such as spaces, hyphens, etc.} 14 | 15 | \item{base_method}{the object of class \code{\linkS4class{Method}} or 16 | of class \code{\linkS4class{Method}} that is being extended} 17 | 18 | \item{extended_method}{a function with arguments "model", "draw", "out", and 19 | "base_method".} 20 | } 21 | \description{ 22 | Creates a new \code{\linkS4class{ExtendedMethod}} object. 23 | } 24 | -------------------------------------------------------------------------------- /man/new_method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/method-class.R 3 | \name{new_method} 4 | \alias{new_method} 5 | \title{Create a Method object} 6 | \usage{ 7 | new_method(name, label, method, settings = list()) 8 | } 9 | \arguments{ 10 | \item{name}{a short name identifier. Must be alphanumeric.} 11 | 12 | \item{label}{a longer, human readable label that can have other characters 13 | such as spaces, hyphens, etc.} 14 | 15 | \item{method}{a function that has arguments "model", "draw" 16 | and (optionally) names matching elements within \code{names(settings)}} 17 | 18 | \item{settings}{(optional) a list of "settings" for the method (e.g., tuning 19 | parameters or related information that might distinguish two otherwise 20 | identical methods).} 21 | } 22 | \description{ 23 | Creates a new \code{\linkS4class{Method}} object. 24 | } 25 | -------------------------------------------------------------------------------- /man/new_method_extension.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/method-extension-class.R 3 | \name{new_method_extension} 4 | \alias{new_method_extension} 5 | \title{Create an object that can be used to make an extended version of a method} 6 | \usage{ 7 | new_method_extension(name, label, method_extension) 8 | } 9 | \arguments{ 10 | \item{name}{a short name identifier. Must be alphanumeric.} 11 | 12 | \item{label}{a longer, human readable label that can have other characters 13 | such as spaces, hyphens, etc.} 14 | 15 | \item{method_extension}{a function with arguments "model", "draw", "out", and 16 | "base_method". This will become the function \code{extended_method} 17 | in the \code{ExtendedMethod} object that is created.} 18 | } 19 | \description{ 20 | Creates an object of class \code{MethodExtension}, which when added to a 21 | \code{Method} creates an \code{\linkS4class{ExtendedMethod}}. 22 | } 23 | \details{ 24 | This class inherits from the \code{\linkS4class{Component}} class. 25 | } 26 | -------------------------------------------------------------------------------- /man/new_metric.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/metric-class.R 3 | \name{new_metric} 4 | \alias{new_metric} 5 | \title{Create a Metric object} 6 | \usage{ 7 | new_metric(name, label, metric) 8 | } 9 | \arguments{ 10 | \item{name}{a short name identifier. Must be alphanumeric.} 11 | 12 | \item{label}{a longer, human readable label that can have other characters 13 | such as spaces, hyphens, etc.} 14 | 15 | \item{metric}{a function with arguments "model" and "out" (and optionally 16 | "draw")} 17 | } 18 | \description{ 19 | Creates a new \code{\linkS4class{Metric}} object. 20 | } 21 | -------------------------------------------------------------------------------- /man/new_model.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/model-class.R 3 | \name{new_model} 4 | \alias{new_model} 5 | \title{Create a Model object} 6 | \usage{ 7 | new_model(name, label, params = list(), simulate) 8 | } 9 | \arguments{ 10 | \item{name}{a short name identifier. Must be alphanumeric (though -, _, and 11 | / are allowed as long as they are not at the start or end of name.} 12 | 13 | \item{label}{a longer, human readable label that can have other characters 14 | such as spaces, hyphens, etc.} 15 | 16 | \item{params}{a list that contains the Model object's parameters} 17 | 18 | \item{simulate}{a function that has arguments \code{nsim} and names 19 | matching elements within \code{names(params)}. It returns a list of 20 | length nsim, where each element of the list represents a random draw 21 | from the \code{Model} object.} 22 | } 23 | \description{ 24 | Creates a new \code{\linkS4class{Model}} object. 25 | } 26 | \examples{ 27 | make_my_example_model <- function(n) { 28 | new_model(name = "normal-data", 29 | label = sprintf("Normal (n = \%s)", n), 30 | params = list(n = n, mu = 2), 31 | simulate = function(n, mu, nsim) { 32 | # this function must return a list of length nsim 33 | x <- matrix(rnorm(n * nsim), n, nsim) 34 | x <- mu + x # true mean is mu 35 | return(split(x, col(x))) # make each col its own list element 36 | }) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /man/new_simulation.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/simulation-class.R 3 | \name{new_simulation} 4 | \alias{new_simulation} 5 | \title{Make a new simulation object} 6 | \usage{ 7 | new_simulation(name, label, dir = ".", refs = list(), save_to_file = TRUE) 8 | } 9 | \arguments{ 10 | \item{name}{a short name identifier. Must be alphanumeric.} 11 | 12 | \item{label}{a longer, human readable label that can have other characters 13 | such as spaces, hyphens, etc.} 14 | 15 | \item{dir}{a directory that reference's directories are relative to} 16 | 17 | \item{refs}{a list containing objects of class \code{\linkS4class{ModelRef}}, 18 | \code{\linkS4class{DrawsRef}}, \code{\linkS4class{OutputRef}}, 19 | and \code{\linkS4class{EvalsRef}}} 20 | 21 | \item{save_to_file}{whether this new simulation should be saved to file. 22 | Default is TRUE. If TRUE, then this simulation can be loaded 23 | in a new R session using \code{dir} and \code{name}.} 24 | } 25 | \description{ 26 | Creates an object of class \code{\linkS4class{Simulation}}. In addition to having 27 | a name and label, this object consists of a set of references to objects of 28 | class \code{\linkS4class{ModelRef}}, \code{\linkS4class{DrawsRef}}, 29 | \code{\linkS4class{OutputRef}}, and \code{\linkS4class{EvalsRef}}. 30 | } 31 | \details{ 32 | A Simulation object is the basic unit of a simulation study. Roughly, one 33 | can think of it as all the files relevant to a single figure. This might be 34 | a single plot or a series of related plots/panels. It could also correspond 35 | to a single table. Note that a Simulation object is light-weight even for 36 | large simulations because it only stores references to the objects 37 | not the objects themselves. The functions \code{\link{model}}, 38 | \code{\link{draws}}, \code{\link{output}}, \code{\link{evals}} can be used 39 | to load individual objects of a simulation. 40 | 41 | The Simulation object created is saved to a file so that it can be loaded 42 | in a new R session. The simulation is saved in dir/files/name.Rdata. Note: 43 | while "files" is the default, the name of this directory is from 44 | getOption("simulator.files"), which is the value of 45 | getOption("simulator.files") when the model was created. 46 | } 47 | \examples{ 48 | sim <- new_simulation(name = "normal-example", 49 | label = "Normal Mean Estimation", 50 | dir = tempdir()) 51 | } 52 | \seealso{ 53 | \code{\link{load_simulation}} \code{\link{save_simulation}} 54 | } 55 | -------------------------------------------------------------------------------- /man/output.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get-from-simulation.R 3 | \name{output} 4 | \alias{output} 5 | \title{Get one or more outputs from a simulation} 6 | \usage{ 7 | output(sim, ..., subset = NULL, index, methods, reference = FALSE) 8 | } 9 | \arguments{ 10 | \item{sim}{a simulation object} 11 | 12 | \item{...}{logical conditions to specify a subset of models. Conditions can 13 | only involve params of model that have length 1 and are of class 14 | numeric or character.} 15 | 16 | \item{subset}{a vector of integers indexing the models or a vector of model 17 | names. To select models based on parameter values, use \code{...}. 18 | However, using \code{...} is slower than using subset.} 19 | 20 | \item{index}{a vector of positive integers specifying which draws' objects 21 | are desired. If missing, then all draws' outputs are returned.} 22 | 23 | \item{methods}{character vector of method names of interest. If missing, 24 | then all methods' outputs are returned} 25 | 26 | \item{reference}{whether to return the ModelRef or the Model object itself} 27 | } 28 | \description{ 29 | Returns either the output object itself or a reference to it. 30 | } 31 | \examples{ 32 | \dontrun{ 33 | # suppose previously we had run the following: 34 | sim <- new_simulation(name = "normal-example", 35 | label = "Normal Mean Estimation", 36 | dir = tempdir()) \%>\% 37 | generate_model(make_my_example_model, n = 20) \%>\% 38 | simulate_from_model(nsim = 50, index = 1:3) \%>\% 39 | run_method(my_example_method) 40 | # then we could get the method's output as follows: 41 | o <- output(sim) 42 | o@out$r1.1 # first random draw's output 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /man/outputs_or_evals.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get-from-simulation.R 3 | \name{outputs_or_evals} 4 | \alias{outputs_or_evals} 5 | \title{Internal function used by both outputs and evals} 6 | \usage{ 7 | outputs_or_evals( 8 | sim, 9 | refs, 10 | sort_by_method, 11 | subset, 12 | index, 13 | methods, 14 | reference, 15 | ... 16 | ) 17 | } 18 | \arguments{ 19 | \item{sim}{simulation object} 20 | 21 | \item{refs}{either sim@output_refs or sim@evals_refs} 22 | 23 | \item{sort_by_method}{whether returned object should have each method's objects 24 | in its own list or not} 25 | } 26 | \description{ 27 | Internal function used by both outputs and evals 28 | } 29 | \keyword{internal} 30 | -------------------------------------------------------------------------------- /man/plot_eval.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/plot_eval.R 3 | \name{plot_eval} 4 | \alias{plot_eval} 5 | \title{Plot a metric's value for each method} 6 | \usage{ 7 | plot_eval( 8 | object, 9 | metric_name, 10 | use_ggplot2 = TRUE, 11 | main, 12 | facet_mains, 13 | ylab, 14 | ylim, 15 | include_zero = FALSE, 16 | angle = 0, 17 | ... 18 | ) 19 | } 20 | \arguments{ 21 | \item{object}{an object of class \code{\linkS4class{Simulation}}, 22 | \code{\linkS4class{Evals}}, or \code{listofEvals}} 23 | 24 | \item{metric_name}{the name of a metric to plot} 25 | 26 | \item{use_ggplot2}{whether to use \code{ggplot2} (requires installation 27 | of \code{ggplot2})} 28 | 29 | \item{main}{title of plot. Default is \code{model_label} when evals is 30 | a single Evals.} 31 | 32 | \item{facet_mains}{only to be used when evals is a \code{listofEvals} 33 | and should be of the same length. Default will be the model_label 34 | for each model.} 35 | 36 | \item{ylab}{the y-axis label (default is \code{metric_label})} 37 | 38 | \item{ylim}{the y-axis limits to use (across all plots)} 39 | 40 | \item{include_zero}{whether ylim should include 0. Ignored if ylim 41 | is passed explicitly} 42 | 43 | \item{angle}{angle of labels (only when \code{use_ggplot2 = FALSE})} 44 | 45 | \item{...}{additional arguments to pass to \code{boxplot} (only when 46 | \code{use_ggplot2 = FALSE}).} 47 | } 48 | \description{ 49 | When the evaluted metric is scalar-valued, this functions makes a boxplot of 50 | this metric for each method. When the metric is vector-valued, this function 51 | makes a curve with this metric on the y-axis, with one curve for each method 52 | (the x-axis is the corresponding entry of that metric's vector). If evals is 53 | a \code{listofEvals}, then each model will be its own plot. 54 | } 55 | \examples{ 56 | \dontrun{ 57 | # suppose previously we had run the following: 58 | sim <- new_simulation(name = "normal-example", 59 | label = "Normal Mean Estimation", 60 | dir = tempdir()) \%>\% 61 | generate_model(make_my_example_model, n = 20) \%>\% 62 | simulate_from_model(nsim = 50, index = 1:3) \%>\% 63 | run_method(my_example_method) \%>\% 64 | evaluate(my_example_loss) 65 | # then we could plot this 66 | plot_eval(sim, "myloss") # "myloss" is my_example_loss@name 67 | } 68 | } 69 | \seealso{ 70 | \code{\link{plot_evals}} \code{\link{plot_eval_by}} 71 | \code{\link{tabulate_eval}} 72 | } 73 | -------------------------------------------------------------------------------- /man/plot_evals.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/plot_evals.R 3 | \name{plot_evals} 4 | \alias{plot_evals} 5 | \title{Plot one metric versus another for each method} 6 | \usage{ 7 | plot_evals( 8 | object, 9 | metric_name_x, 10 | metric_name_y, 11 | use_ggplot2 = TRUE, 12 | main, 13 | facet_mains, 14 | xlab, 15 | ylab, 16 | xlim, 17 | ylim, 18 | include_zero = FALSE, 19 | legend_location = "topright", 20 | method_col = seq(num_methods), 21 | method_lty = rep(1, num_methods), 22 | method_lwd = rep(1, num_methods), 23 | method_pch = rep(NA, num_methods), 24 | ... 25 | ) 26 | } 27 | \arguments{ 28 | \item{object}{an object of class \code{\linkS4class{Simulation}}, 29 | \code{\linkS4class{Evals}}, or \code{listofEvals}} 30 | 31 | \item{metric_name_x}{the name of metric to plot on x axis (or NULL)} 32 | 33 | \item{metric_name_y}{the name of metric to plot on y axis} 34 | 35 | \item{use_ggplot2}{whether to use \code{ggplot2} (requires installation 36 | of \code{ggplot2})} 37 | 38 | \item{main}{title of plot. Default is \code{model_label} when evals is 39 | a single Evals.} 40 | 41 | \item{facet_mains}{only to be used when evals is a \code{listofEvals} 42 | and should be of the same length. Default will be the model_label 43 | for each model.} 44 | 45 | \item{xlab}{the x-axis label (default is \code{metric_label_x})} 46 | 47 | \item{ylab}{the y-axis label (default is \code{metric_label_y})} 48 | 49 | \item{xlim}{the limits of the x-axis} 50 | 51 | \item{ylim}{the limits of the y-axis} 52 | 53 | \item{include_zero}{whether ylim should include 0. Ignored if ylim 54 | is passed explicitly} 55 | 56 | \item{legend_location}{location of legend. Set to NULL to remove legend.} 57 | 58 | \item{method_col}{color to use for each method} 59 | 60 | \item{method_lty}{line style to use for each method} 61 | 62 | \item{method_lwd}{line thickness to use for each method} 63 | 64 | \item{method_pch}{point style to use for each method (default is that no 65 | points, only lines are drawn)} 66 | 67 | \item{...}{additional arguments to pass to \code{boxplot} (only when 68 | \code{use_ggplot2 = FALSE}).} 69 | } 70 | \description{ 71 | This function is used when both evaluated metrics are vector-valued, so a 72 | curve is plotted, parametrized by the two metrics. To plot a single metric 73 | that is vector-valued, pass NULL for metric_name_x. This behaves similarly 74 | to \code{plot(runif(5))}, in which the x-axis variable is simply \code{1:5}. 75 | If evals is a \code{listofEvals}, then each model will be its own plot. 76 | } 77 | -------------------------------------------------------------------------------- /man/plus-ExtendedMethod-MethodExtension-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/method-extension-class.R 3 | \name{+,ExtendedMethod,MethodExtension-method} 4 | \alias{+,ExtendedMethod,MethodExtension-method} 5 | \title{Create an ExtendedMethod from an ExtendedMethod and MethodExtension} 6 | \usage{ 7 | \S4method{+}{ExtendedMethod,MethodExtension}(e1, e2) 8 | } 9 | \arguments{ 10 | \item{e1}{an object of class \code{\linkS4class{ExtendedMethod}}} 11 | 12 | \item{e2}{an object of class \code{\linkS4class{MethodExtension}}} 13 | } 14 | \description{ 15 | Create an ExtendedMethod from an ExtendedMethod and MethodExtension 16 | } 17 | -------------------------------------------------------------------------------- /man/plus-Method-MethodExtension-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/method-extension-class.R 3 | \name{+,Method,MethodExtension-method} 4 | \alias{+,Method,MethodExtension-method} 5 | \title{Create an ExtendedMethod from a Method and MethodExtension} 6 | \usage{ 7 | \S4method{+}{Method,MethodExtension}(e1, e2) 8 | } 9 | \arguments{ 10 | \item{e1}{an object of class \code{\linkS4class{Method}}} 11 | 12 | \item{e2}{an object of class \code{\linkS4class{MethodExtension}}} 13 | } 14 | \description{ 15 | Create an ExtendedMethod from a Method and MethodExtension 16 | } 17 | -------------------------------------------------------------------------------- /man/plus-list-MethodExtension-method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/method-extension-class.R 3 | \name{+,list,MethodExtension-method} 4 | \alias{+,list,MethodExtension-method} 5 | \title{Create a list of ExtendedMethod from a list of Methods and a MethodExtension} 6 | \usage{ 7 | \S4method{+}{list,MethodExtension}(e1, e2) 8 | } 9 | \arguments{ 10 | \item{e1}{a list of objects of class \code{\linkS4class{Method}} or of class 11 | \code{\linkS4class{ExtendedMethod}}} 12 | 13 | \item{e2}{an object of class \code{\linkS4class{MethodExtension}}} 14 | } 15 | \description{ 16 | Create a list of ExtendedMethod from a list of Methods and a MethodExtension 17 | } 18 | -------------------------------------------------------------------------------- /man/recycle.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/plot_eval_by.R 3 | \name{recycle} 4 | \alias{recycle} 5 | \title{Recycles elements to create vector of desired length} 6 | \usage{ 7 | recycle(x, length) 8 | } 9 | \arguments{ 10 | \item{x}{vector to be expanded to proper length} 11 | 12 | \item{length}{desired length} 13 | } 14 | \description{ 15 | Recycles elements to create vector of desired length 16 | } 17 | -------------------------------------------------------------------------------- /man/relabel.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/simulation-class.R 3 | \name{relabel} 4 | \alias{relabel} 5 | \title{Give simulation a new label} 6 | \usage{ 7 | relabel(sim, label) 8 | } 9 | \arguments{ 10 | \item{sim}{object of class \code{\linkS4class{Simulation}}} 11 | 12 | \item{label}{a longer, human readable label that can have other characters 13 | such as spaces, hyphens, etc.} 14 | } 15 | \description{ 16 | Note that \code{\link{save_simulation}} needs to be called for this change 17 | to be saved to file. 18 | } 19 | \seealso{ 20 | \code{\link{rename}} 21 | } 22 | -------------------------------------------------------------------------------- /man/rename.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/simulation-class.R 3 | \name{rename} 4 | \alias{rename} 5 | \title{Give simulation a new name} 6 | \usage{ 7 | rename(sim, name) 8 | } 9 | \arguments{ 10 | \item{sim}{object of class \code{\linkS4class{Simulation}}} 11 | 12 | \item{name}{a short name identifier. Must be an alphanumeric (but can also 13 | have - or _ within} 14 | } 15 | \description{ 16 | Note that \code{\link{save_simulation}} needs to be called for this change 17 | to be saved to file. 18 | } 19 | \seealso{ 20 | \code{\link{relabel}} 21 | } 22 | -------------------------------------------------------------------------------- /man/run_extendedmethod_single.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/methods.R 3 | \name{run_extendedmethod_single} 4 | \alias{run_extendedmethod_single} 5 | \title{Run a single extended method on a single index of simulated data.} 6 | \usage{ 7 | run_extendedmethod_single(extmethod, model, draws, base_output_list) 8 | } 9 | \arguments{ 10 | \item{extmethod}{a \code{\linkS4class{ExtendedMethod}} object} 11 | 12 | \item{model}{a \code{\linkS4class{Model}} object} 13 | 14 | \item{draws}{a \code{\linkS4class{Draws}} object generated by \code{model}} 15 | 16 | \item{base_output_list}{the result of loading a \code{\linkS4class{Output}} object with 17 | \code{more_info = TRUE} so that it includes RNG endstate.} 18 | } 19 | \description{ 20 | This is an internal function. Users should call the wrapper function. 21 | \code{\link{run_method}}. Here "single" refers to a single 22 | index-ExtendedMethod 23 | pair. 24 | } 25 | -------------------------------------------------------------------------------- /man/run_extmethod_parallel.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/parallel-methods.R 3 | \name{run_extmethod_parallel} 4 | \alias{run_extmethod_parallel} 5 | \title{Run one or more extended methods on simulated data.} 6 | \usage{ 7 | run_extmethod_parallel( 8 | my_extmethods, 9 | dir, 10 | model_name, 11 | index, 12 | out_dir, 13 | out_loc, 14 | socket_names, 15 | libraries, 16 | save_locally = TRUE 17 | ) 18 | } 19 | \arguments{ 20 | \item{my_extmethods}{list of extended methods to be run in parallel} 21 | 22 | \item{dir}{the directory where Model object was saved (by 23 | \code{\link{generate_model}})} 24 | 25 | \item{model_name}{the Model object's \code{name} attribute} 26 | 27 | \item{index}{a vector of positive integer indices. Allows simulations to be 28 | carried out in chunks. Each chunk gets a separate RNG stream, 29 | meaning that the results will be identical whether we run these in 30 | parallel or sequentially.} 31 | 32 | \item{out_dir}{full directory to where method outputs are stored.} 33 | 34 | \item{out_loc}{a length-1 character vector that gives location 35 | (relative to model's path) that method outputs are stored.This can be 36 | useful for staying organized when multiple simulations are based on 37 | the same Model and Draws objects.} 38 | 39 | \item{socket_names}{(quoting from \code{\link[parallel]{makePSOCKcluster}}) 40 | "either a character vector of host names on which to run the worker 41 | copies of R, or a positive integer (in which case that number of 42 | copies is run on localhost)."} 43 | 44 | \item{libraries}{character vector of R packages that will be needed on the 45 | slaves.} 46 | 47 | \item{save_locally}{if TRUE, then files will be saved on slaves. If FALSE, 48 | they will be saved on master.} 49 | } 50 | \description{ 51 | This is an internal function. Jobs are run in parallel both by method 52 | and by index. Users should call the wrapper function 53 | \code{\link{run_method}}. 54 | } 55 | \keyword{internal} 56 | -------------------------------------------------------------------------------- /man/run_method.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/methods.R 3 | \name{run_method} 4 | \alias{run_method} 5 | \title{Run one or more methods on simulated data.} 6 | \usage{ 7 | run_method(object, methods, out_loc = "out", parallel = NULL) 8 | } 9 | \arguments{ 10 | \item{object}{an object of class \code{\linkS4class{DrawsRef}} (or a list of 11 | such objects) as returned by \code{link{simulate_from_model}}. If 12 | \code{object} is a \code{\linkS4class{Simulation}}, then function is applied 13 | to the referenced draws in that simulation and returns the same 14 | \code{Simulation} object but with references added to the new outputs 15 | created.} 16 | 17 | \item{methods}{a list of \code{\linkS4class{Method}} and/or 18 | \code{\linkS4class{ExtendedMethod}} objects or a single \code{\linkS4class{Method}} 19 | or object \code{\linkS4class{ExtendedMethod}}} 20 | 21 | \item{out_loc}{(optional) a length-1 character vector that gives location 22 | (relative to model's path) that method outputs are stored.This can be 23 | useful for staying organized when multiple simulations are based on 24 | the same Model and Draws objects.} 25 | 26 | \item{parallel}{either \code{NULL} or a list containing \code{socket_names} 27 | and (optionally) \code{libraries} and \code{save_locally} 28 | (see Details for more information)} 29 | } 30 | \description{ 31 | Given a \code{\linkS4class{Method}} object or list of \code{\linkS4class{Method}} objects, 32 | this function runs the method(s) on the draws passed through \code{object}. 33 | The output of each method is saved to file. 34 | } 35 | \details{ 36 | This function creates objects of class \code{\linkS4class{Output}} and saves each to 37 | file (at dir/model_name//r_.Rdata. If parallel 38 | is not NULL, then it must be a list containing \code{socket_names}, which can 39 | either be a positive integer specifying the number of copies to run on 40 | localhost or else a character vector of machine names (e.g., 41 | "mycluster-0-0"). The list \code{parallel} can also contain 42 | \code{libraries}, a character vector of R packages that will be needed on the 43 | slaves and \code{save_locally}, a logical that indicates whether the files 44 | generated should be saved on the slaves (i.e., locally) or on the master. 45 | 46 | Before running each method on index i, the RNG state is restored to what it 47 | was at the end of calling \code{\link{simulate_from_model}} on this index. 48 | This is only relevant for randomized methods. The choice to do this ensures 49 | that one will get identical results regardless of the order in which methods 50 | and indices are run in. When \code{\linkS4class{ExtendedMethod}} objects are 51 | passed, these are run after all \code{Method} objects have been run. This is 52 | because each \code{ExtendedMethod} object depends on the output of its base 53 | method. Furthermore, before an \code{ExtendedMethod} is called, the RNG 54 | state is restored to what it was after the base method had been called. 55 | } 56 | \examples{ 57 | \dontrun{ 58 | # suppose previously we had run the following: 59 | sim <- new_simulation(name = "normal-example", 60 | label = "Normal Mean Estimation", 61 | dir = tempdir()) \%>\% 62 | generate_model(make_my_example_model, n = 20) \%>\% 63 | simulate_from_model(nsim = 50, index = 1:3) 64 | # then we could add 65 | sim <- run_method(sim, my_example_method) 66 | } 67 | } 68 | \seealso{ 69 | \code{\link{generate_model}} \code{\link{simulate_from_model}} 70 | } 71 | -------------------------------------------------------------------------------- /man/run_method_parallel.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/parallel-methods.R 3 | \name{run_method_parallel} 4 | \alias{run_method_parallel} 5 | \title{Run one or more methods on simulated data.} 6 | \usage{ 7 | run_method_parallel( 8 | my_methods, 9 | dir, 10 | model_name, 11 | index, 12 | out_dir, 13 | out_loc, 14 | socket_names, 15 | libraries, 16 | save_locally = TRUE 17 | ) 18 | } 19 | \arguments{ 20 | \item{my_methods}{list of methods to be run in parallel} 21 | 22 | \item{dir}{the directory where Model object was saved (by 23 | \code{\link{generate_model}})} 24 | 25 | \item{model_name}{the Model object's \code{name} attribute} 26 | 27 | \item{index}{a vector of positive integer indices. Allows simulations to be 28 | carried out in chunks. Each chunk gets a separate RNG stream, 29 | meaning that the results will be identical whether we run these in 30 | parallel or sequentially.} 31 | 32 | \item{out_dir}{full directory to where method outputs are stored.} 33 | 34 | \item{out_loc}{a length-1 character vector that gives location 35 | (relative to model's path) that method outputs are stored.This can be 36 | useful for staying organized when multiple simulations are based on 37 | the same Model and Draws objects.} 38 | 39 | \item{socket_names}{(quoting from \code{\link[parallel]{makePSOCKcluster}} 40 | "either a character vector of host names on which to run the worker 41 | copies of R, or a positive integer (in which case that number of 42 | copies is run on localhost)."} 43 | 44 | \item{libraries}{character vector of R packages that will be needed on the 45 | slaves.} 46 | 47 | \item{save_locally}{if TRUE, then files will be saved on slaves. If FALSE, 48 | they will be saved on master.} 49 | } 50 | \description{ 51 | This is an internal function. Jobs are run in parallel both by method 52 | and by index. Users should call the wrapper function 53 | \code{\link{run_method}}. 54 | } 55 | \keyword{internal} 56 | -------------------------------------------------------------------------------- /man/run_method_single.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/methods.R 3 | \name{run_method_single} 4 | \alias{run_method_single} 5 | \title{Run a single method on a single index of simulated data.} 6 | \usage{ 7 | run_method_single(method, model, draws_list) 8 | } 9 | \arguments{ 10 | \item{method}{a \code{\linkS4class{Method}} object} 11 | 12 | \item{model}{a \code{\linkS4class{Model}} object} 13 | 14 | \item{draws_list}{the result of loading a \code{\linkS4class{Draws}} object with 15 | \code{more_info = TRUE} so that it includes RNG endstate.} 16 | } 17 | \description{ 18 | This is an internal function. Users should call the wrapper function. 19 | \code{\link{run_method}}. Here "single" refers to a single index-method 20 | pair. 21 | } 22 | -------------------------------------------------------------------------------- /man/save_simulation.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/simulation-class.R 3 | \name{save_simulation} 4 | \alias{save_simulation} 5 | \title{Save a simulation object} 6 | \usage{ 7 | save_simulation(sim) 8 | } 9 | \arguments{ 10 | \item{sim}{an object of class \code{\linkS4class{Simulation}}} 11 | } 12 | \description{ 13 | Saves an object of class \code{\linkS4class{Simulation}} to 14 | sim@dir/files/sim@name.Rdata. Note: while "files" is the default, the name 15 | of this directory is from getOption("simulator.files"), which is the value of 16 | getOption("simulator.files") when the model was created. 17 | } 18 | \details{ 19 | This function overwrites any pre-existing file in that location without 20 | apology. 21 | } 22 | \seealso{ 23 | \code{\link{new_simulation}} \code{\link{load_simulation}} 24 | } 25 | -------------------------------------------------------------------------------- /man/simulate_from_model.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/draws.R 3 | \name{simulate_from_model} 4 | \alias{simulate_from_model} 5 | \title{Simulate from a model.} 6 | \usage{ 7 | simulate_from_model(object, nsim, index = 1, parallel = NULL) 8 | } 9 | \arguments{ 10 | \item{object}{an object of class \code{\linkS4class{ModelRef}} as returned by 11 | \code{link{generate_model}}. Or a list of such objects. If 12 | \code{object} is a \code{Simulation}, then function is applied to the 13 | referenced models in that simulation and returns the same 14 | \code{Simulation} object but with references added to the new draws 15 | created.} 16 | 17 | \item{nsim}{number of simulations to be conducted. If a scalar, then 18 | value repeated for each index. Otherwise can be a vector of length 19 | \code{length(index)}} 20 | 21 | \item{index}{a vector of positive integer indices. Allows simulations to be 22 | carried out in chunks. Each chunk gets a separate RNG stream, 23 | meaning that the results will be identical whether we run these in 24 | parallel or sequentially.} 25 | 26 | \item{parallel}{either \code{NULL} or a list containing \code{socket_names} 27 | and (optionally) \code{libraries} and \code{save_locally} 28 | (see Details for more information)} 29 | } 30 | \description{ 31 | Given a reference to a \code{\linkS4class{Model}} object, this function calls the 32 | model's \code{simulate} function on its \code{params}. It repeats this 33 | \code{nsim} times. For example, when simulating regression with a fixed 34 | design, this function would generate \code{nsim} response vectors \code{y}. 35 | } 36 | \details{ 37 | This function creates objects of class \code{\linkS4class{Draws}} and saves each to 38 | file (at dir/files/model_name/r.Rdata). Note: while "files" is the 39 | default, the name of this directory is from getOption("simulator.files"), 40 | which is the value of getOption("simulator.files") when the model was 41 | created. 42 | 43 | If parallel is not NULL, then it must be a list containing 44 | \code{socket_names}, which can either be a positive integer specifying the 45 | number of copies to run on localhost or else a character vector of machine 46 | names (e.g., "mycluster-0-0"). The list \code{parallel} can also contain 47 | \code{libraries}, a character vector of R packages that will be needed on the 48 | slaves and \code{save_locally}, a logical that indicates whether the files 49 | generated should be saved on the slaves (i.e., locally) or on the master. 50 | } 51 | \examples{ 52 | \dontrun{ 53 | sim <- new_simulation(name = "normal-example", 54 | label = "Normal Mean Estimation", 55 | dir = tempdir()) \%>\% 56 | generate_model(make_my_example_model, n = 20) \%>\% 57 | simulate_from_model(nsim = 50, index = 1:3, 58 | parallel = list(socket_names = 3)) 59 | } 60 | } 61 | \seealso{ 62 | \code{\link{load_draws}} \code{\link{generate_model}} \code{\link{run_method}} 63 | } 64 | -------------------------------------------------------------------------------- /man/simulate_from_model_single.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/draws.R 3 | \name{simulate_from_model_single} 4 | \alias{simulate_from_model_single} 5 | \title{Simulate from a model.} 6 | \usage{ 7 | simulate_from_model_single(model, nsim, index, seed) 8 | } 9 | \arguments{ 10 | \item{model}{a Model object} 11 | 12 | \item{nsim}{number of simulations to be conducted.} 13 | 14 | \item{index}{a positive integer index.} 15 | 16 | \item{seed}{this is the 7 digit seed used by L'Ecuyer RNG} 17 | } 18 | \description{ 19 | This is an internal function. Users should call the wrapper function 20 | \code{\link{simulate_from_model}}. 21 | } 22 | -------------------------------------------------------------------------------- /man/simulate_parallel.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/parallel-draws.R 3 | \name{simulate_parallel} 4 | \alias{simulate_parallel} 5 | \title{Simulate from a model in parallel.} 6 | \usage{ 7 | simulate_parallel( 8 | model_ref, 9 | nsim, 10 | index, 11 | seeds, 12 | socket_names, 13 | libraries, 14 | save_locally = TRUE 15 | ) 16 | } 17 | \arguments{ 18 | \item{model_ref}{object of class \code{\linkS4class{ModelRef}}} 19 | 20 | \item{nsim}{number of simulations to be conducted on each chunk. Vector of 21 | same length as \code{index}} 22 | 23 | \item{index}{a vector of positive integer indices. Allows simulations to be 24 | carried out in chunks. Each chunk gets a separate RNG stream, 25 | meaning that the results will be identical whether we run these in 26 | parallel or sequentially.} 27 | 28 | \item{seeds}{a list of \code{length(index)} L'Ecuyer-CMRG seed vectors. 29 | Each should be from a separate stream. In particular, starting from 30 | the seed used to generate the model object, seeds[i] should be the 31 | result of calling \code{\link[parallel]{nextRNGStream}} index[i] 32 | times.} 33 | 34 | \item{socket_names}{(quoting from \code{\link[parallel]{makePSOCKcluster}} 35 | "either a character vector of host names on which to run the worker 36 | copies of R, or a positive integer (in which case that number of 37 | copies is run on localhost)."} 38 | 39 | \item{libraries}{character vector of R packages that will be needed on the 40 | slaves.} 41 | 42 | \item{save_locally}{if TRUE, then files will be saved on slaves. If FALSE, 43 | they will be saved on master.} 44 | } 45 | \description{ 46 | This is an internal function. Draws are done in chunks labeled 47 | by indices and of size determined by nsim. Users should call the wrapper 48 | function \code{\link{simulate_from_model}}. 49 | } 50 | -------------------------------------------------------------------------------- /man/subset_evals.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/evaluate.R 3 | \name{subset_evals} 4 | \alias{subset_evals} 5 | \title{Reduce an Evals object to a subset of methods and/or metrics} 6 | \usage{ 7 | subset_evals(evals, method_names = NULL, metric_names = NULL) 8 | } 9 | \arguments{ 10 | \item{evals}{an object of class \code{\linkS4class{Evals}} or 11 | \code{listofEvals}.} 12 | 13 | \item{method_names}{a character vector of method names} 14 | 15 | \item{metric_names}{a character vector of metric names} 16 | } 17 | \description{ 18 | If \code{method_names} is \code{NULL}, then subsetting is not done over 19 | methods. Likewise for \code{metric_names}. 20 | } 21 | -------------------------------------------------------------------------------- /man/subset_models.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/model-class.R 3 | \name{subset_models} 4 | \alias{subset_models} 5 | \title{Subset Models} 6 | \usage{ 7 | subset_models(m, ...) 8 | } 9 | \arguments{ 10 | \item{m}{list of \code{\linkS4class{Model}} objects} 11 | 12 | \item{...}{logical expression involving parameters of Models. For now, can 13 | only be parameters that are of length 1 and either of class numeric or 14 | character} 15 | } 16 | \description{ 17 | Given a list of \code{\linkS4class{Model}} objects, returns model names 18 | which meet conditions. Uses \code{\link{subset}} 19 | } 20 | -------------------------------------------------------------------------------- /man/subset_simulation.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get-from-simulation.R 3 | \name{subset_simulation} 4 | \alias{subset_simulation} 5 | \title{Create a simulation that is a subset of a preexisting simulation object} 6 | \usage{ 7 | subset_simulation(sim, ..., subset = NULL, index, methods) 8 | } 9 | \arguments{ 10 | \item{sim}{a simulation object} 11 | 12 | \item{...}{logical conditions to specify a subset of models. Conditions can 13 | only involve params of model that have length 1 and are of class 14 | numeric or character.} 15 | 16 | \item{subset}{a vector of integers indexing the models or a vector of model 17 | names. To select models based on parameter values, use \code{...}. 18 | However, using \code{...} is slower than using subset.} 19 | 20 | \item{index}{a vector of positive integers specifying which draws' objects 21 | are desired. If missing, then all draws' evals are returned.} 22 | 23 | \item{methods}{character vector of method names of interest. If missing, 24 | then all methods' evals are returned} 25 | } 26 | \description{ 27 | Given a simulation, creates a new simulation that is a subset of the 28 | preexisting simulation. Does not save this new one to file. To do so, 29 | first change the name (and, potentially, label) of the simulation 30 | and then use \code{\link{save_simulation}}. If you call 31 | \code{\link{save_simulation}} before changing the name, you will overwrite 32 | the preexisting simulation. Use \code{\link{rename}} and 33 | \code{\link{relabel}}. 34 | } 35 | -------------------------------------------------------------------------------- /man/tabulate_eval.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/tables.R 3 | \name{tabulate_eval} 4 | \alias{tabulate_eval} 5 | \title{Make a table of a metric for each pair of models and methods} 6 | \usage{ 7 | tabulate_eval( 8 | object, 9 | metric_name, 10 | method_names = NULL, 11 | caption = NULL, 12 | center_aggregator = NULL, 13 | spread_aggregator = NULL, 14 | se_format = c("Paren", "PlusMinus", "None"), 15 | output_type = "latex", 16 | format_args = list(nsmall = 0, digits = NULL, scientific = FALSE), 17 | na_string = "--", 18 | bold = c("None", "Smallest", "Largest") 19 | ) 20 | } 21 | \arguments{ 22 | \item{object}{an object of class \code{\linkS4class{Simulation}}, 23 | \code{\linkS4class{Evals}}, or \code{listofEvals}. 24 | Each evals object should just differ by model_name.} 25 | 26 | \item{metric_name}{the name of a metric to tabulate. Must be scalar valued.} 27 | 28 | \item{method_names}{character vector indicating methods to include in table. 29 | If NULL, then will include all methods found in object's evals.} 30 | 31 | \item{caption}{caption of plot. If NULL, then default caption used; if FALSE 32 | then no caption (and returns tabular without table).} 33 | 34 | \item{center_aggregator}{When NULL (which is default), the sample mean 35 | aggregator is used. User can write specialized aggregators (see 36 | definition of class \code{\linkS4class{Aggregator}}) as necessary, 37 | for example, when the evaluated metric is not scalar-valued.} 38 | 39 | \item{spread_aggregator}{When NULL (which is default), the standard error 40 | of the sample mean is used. User can write specialized aggregators (see 41 | definition of class \code{\linkS4class{Aggregator}}) as necessary, 42 | for example, when the evaluated metric is not scalar-valued. Set 43 | \code{spread_aggregator} to \code{NA} to hide error bars.} 44 | 45 | \item{se_format}{format of the standard error} 46 | 47 | \item{output_type}{see \code{\link[knitr]{kable}}'s argument format for options. 48 | Default is "latex" but other options include "html" and "markdown"} 49 | 50 | \item{format_args}{arguments to pass to the function \code{\link{format}}} 51 | 52 | \item{na_string}{what to write in table in place of NA} 53 | 54 | \item{bold}{puts in bold the value that is smallest/largest for each model} 55 | } 56 | \description{ 57 | Each row of the table corresponds to a different model and each column 58 | to a different method. The metric must be a scalar. The way in which 59 | standard error is shown (or not shown) is controlled by \code{se_format}. 60 | } 61 | \details{ 62 | Uses \code{knitr}'s function \code{kable} to put table in various formats, 63 | including latex, html, markdown, etc. 64 | } 65 | \examples{ 66 | \dontrun{ 67 | # suppose previously we had run the following: 68 | sim <- new_simulation(name = "normal-example", 69 | label = "Normal Mean Estimation", 70 | dir = tempdir()) \%>\% 71 | generate_model(make_my_example_model, 72 | n = list(10, 20, 30), 73 | vary_along = "n") \%>\% 74 | simulate_from_model(nsim = 50, index = 1:3) \%>\% 75 | run_method(my_example_method) \%>\% 76 | evaluate(my_example_loss) 77 | # then we could plot this 78 | tabulate_eval(sim, "myloss") 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /simulator.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: No 4 | SaveWorkspace: No 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: knitr 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | 18 | BuildType: Package 19 | PackageUseDevtools: Yes 20 | PackageInstallArgs: --no-multiarch --with-keep.source 21 | PackageRoxygenize: rd,collate,namespace 22 | -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- 1 | # Got this Sys.setenv line from https://github.com/luckyrandom/cmaker/blob/b85813ac2b7aef69932eca8fbb4fa0ec225e0af0/tests/testthat.R 2 | # so that devtools:check() won't hang. 3 | # setting R_TESTS to empty string because of 4 | # https://github.com/hadley/testthat/issues/144 5 | # revert this when that issue in R is fixed. 6 | Sys.setenv("R_TESTS" = "") 7 | 8 | library(testthat) 9 | library(simulator) 10 | 11 | test_check("simulator") 12 | -------------------------------------------------------------------------------- /tests/testthat/test-create.R: -------------------------------------------------------------------------------- 1 | options(simulator.verbose = FALSE) 2 | 3 | context("create_template") 4 | 5 | test_that("create makes a template that can be run without error", { 6 | dir <- file.path(tempdir(), "example") 7 | if (!dir.exists(dir)) dir.create(dir) 8 | expect_error(create(file.path(dir, "hello")), NA) 9 | knitr::opts_knit$set(base.dir = file.path(dir, "hello")) 10 | a <- knitr::knit(file.path(dir, "hello", "writeup.Rmd"), 11 | output = file.path(dir, "hello/writeup.md"), 12 | quiet = TRUE) 13 | expect_false(any(grepl("error", readLines(a), ignore.case = TRUE))) 14 | # let's check that running twice gives same answer (testing caching logic) 15 | b <- knitr::knit(file.path(dir, "hello", "writeup.Rmd"), 16 | output = file.path(dir, "hello/writeup_2ndtime.md"), 17 | quiet = TRUE) 18 | # only the "generated on" line should differ (depending on time) 19 | aa <- readLines(a) 20 | bb <- readLines(b) 21 | expect_true(sum(aa != bb) %in% c(0, 1)) 22 | unlink(dir, recursive = TRUE) 23 | }) 24 | -------------------------------------------------------------------------------- /tests/testthat/test-draws.R: -------------------------------------------------------------------------------- 1 | options(simulator.verbose = FALSE) 2 | 3 | context("draws related") 4 | 5 | make_regmodel2 <- function(n, p, sigma) { 6 | x <- matrix(rnorm(n * p), n, p) 7 | beta <- runif(p) 8 | return(new_model(name = "reg2", 9 | label = sprintf("Regression model"), 10 | params = list(x = x, beta = beta, signal = x %*% beta, 11 | n = n, sigma = sigma), 12 | simulate = function(signal, n, sigma, nsim) { 13 | y <- list() 14 | for (i in seq(nsim)) 15 | y[[i]] <- signal + sigma * rnorm(n) 16 | return(y) 17 | })) 18 | } 19 | 20 | test_that("loading DrawsRef works (even with changing simulator.files)", { 21 | dir <- file.path(tempdir(), "example") 22 | if (!dir.exists(dir)) dir.create(dir) 23 | options(simulator.files = "hello") 24 | mref <- generate_model(dir, make_regmodel2, n = 5, p = 2, sigma = 2) 25 | dref <- simulate_from_model(mref, nsim = 2, index = 1:3) 26 | options(simulator.files = "files") 27 | draws1 <- load_draws(dir, "reg2/n_5/p_2/sigma_2", 3, 28 | simulator.files = "hello") 29 | draws2 <- load(dref[[3]]) 30 | expect_identical(draws1, draws2) 31 | options(simulator.files = "files") # reset 32 | unlink(dir, recursive = TRUE) 33 | }) 34 | 35 | test_that("loading multiple indices versus single gives same results", { 36 | dir <- file.path(tempdir(), "example") 37 | if (!dir.exists(dir)) dir.create(dir) 38 | mref <- generate_model(dir, make_regmodel2, n = 5, p = 2, sigma = 2) 39 | dref <- simulate_from_model(mref, nsim = 2, index = 1:3) 40 | draws <- load_draws(dir, "reg2/n_5/p_2/sigma_2", index = 1:3, 41 | more_info = TRUE) 42 | draws2 <- load_draws(dir, "reg2/n_5/p_2/sigma_2", index = 2, 43 | more_info = TRUE) 44 | expect_identical(draws$rng[[2]], draws2$rng) 45 | unlink(dir, recursive = TRUE) 46 | }) 47 | -------------------------------------------------------------------------------- /tests/testthat/test-evaluate.R: -------------------------------------------------------------------------------- 1 | options(simulator.verbose = FALSE) 2 | 3 | context("evaluate") 4 | 5 | make_testmodel <- function() { 6 | return(new_model(name = "tm", 7 | label = sprintf("Test model"), 8 | params = list(n = 2, x = runif(2)), 9 | simulate = function(x, n, nsim) { 10 | y <- list() 11 | for (i in seq(nsim)) 12 | y[[i]] <- x + rnorm(n) 13 | return(y) 14 | })) 15 | } 16 | 17 | his_method <- new_method(name = "his", 18 | label = "His method", 19 | method = function(model, draw) { 20 | return(list(est = median(draw))) 21 | }) 22 | my_method <- new_method(name = "my", 23 | label = "My method", 24 | method = function(model, draw) { 25 | return(list(est = mean(draw), 26 | f = draw[1])) 27 | }) 28 | 29 | squared_error <- new_metric(name = "se", 30 | label = "Squared Error", 31 | metric = function(model, out) { 32 | sum((out$est - model$x)^2) 33 | }) 34 | 35 | l1_error <- new_metric(name = "l1", 36 | label = "L1 Error", 37 | metric = function(model, out) { 38 | sum(abs(out$est - model$x)) 39 | }) 40 | 41 | linf_error <- new_metric(name = "li", 42 | label = "L Infinity Error", 43 | metric = function(model, out) { 44 | max(abs(out$est - model$x)) 45 | }) 46 | 47 | normy <- new_metric(name = "normy", 48 | label = "Norm of y", 49 | metric = function(model, out, draw) { 50 | sqrt(sum(draw^2)) 51 | }) 52 | 53 | test_that("evaluate handles arguments as desired", { 54 | dir <- file.path(tempdir(), "example") 55 | if (!dir.exists(dir)) dir.create(dir) 56 | mref <- generate_model(dir, make_testmodel) 57 | dref <- simulate_from_model(mref, nsim = 20, 1:3) 58 | oref <- run_method(dref, list(my_method, his_method)) 59 | eref <- evaluate(oref, list(squared_error, l1_error, linf_error)) 60 | expect_identical(load(eref[c(1,3,5)]), load_evals(dir, "tm", 1:3, "my")) 61 | evals2a <- load_evals(dir, "tm", 2, c("my", "his"), metric_names = "l1") 62 | evals12a <- load_evals(dir, "tm", 1:2, c("my", "his"), metric_names = "l1") 63 | evaluate(oref, l1_error) 64 | evals2b <- load_evals(dir, "tm", 2, "his", metric_names = "l1") 65 | expect_identical(subset_evals(evals2a, method_names = "his"), evals2b) 66 | evaluate(oref, l1_error) 67 | evals12b <- load_evals(dir, "tm", 1:2, "his", metric_names = "l1") 68 | expect_identical(subset_evals(evals12a, method = "his"), evals12b) 69 | evaluate(oref, list(l1_error, linf_error, normy)) 70 | unlink(dir, recursive = TRUE) 71 | }) 72 | -------------------------------------------------------------------------------- /tests/testthat/test-method-extension.R: -------------------------------------------------------------------------------- 1 | options(simulator.verbose = FALSE) 2 | 3 | context("method-extension") 4 | 5 | make_testmodel <- function() { 6 | return(new_model(name = "tm", 7 | label = "Test model", 8 | params = list(n = 20, mu = rep(c(0, 2), c(15, 5))), 9 | simulate = function(mu, n, nsim) { 10 | y <- list() 11 | for (i in seq(nsim)) 12 | y[[i]] <- mu + 0.5 * rnorm(n) 13 | return(y) 14 | })) 15 | } 16 | 17 | soft <- new_method("st", "Soft threshold", 18 | method = function(model, draw) { 19 | lamlist <- seq(max(model$mu), 0, length = 10) 20 | st <- function(lam) sign(draw) * pmax(draw - lam, 0) 21 | list(fit = sapply(lamlist, st)) 22 | }) 23 | 24 | refit <- new_method_extension(name = "refit", label = "refitted", 25 | method_extension = function(model, draw, out, 26 | base_method) { 27 | fit <- apply(out$fit, 2, function(a) { 28 | ii <- which(a != 0) 29 | a[ii] <- draw[ii] 30 | return(a) 31 | }) 32 | list(fit = fit) 33 | }) 34 | 35 | mult_by_2 <- new_method_extension(name = "multby2", label = "doubled", 36 | method_extension = function(model, draw, out, 37 | base_method) { 38 | out$fit <- 2 * out$fit 39 | list(fit = out$fit) 40 | }) 41 | 42 | mse <- new_metric("mse", "MSE", metric = function(model, out) { 43 | colSums(as.matrix((model$mu - out$fit)^2)) 44 | }) 45 | 46 | 47 | test_that("extend a method", { 48 | dir <- file.path(tempdir(), "example") 49 | if (!dir.exists(dir)) dir.create(dir) 50 | sim <- new_simulation("sparse_vector", "Estimation of a sparse vector", dir = dir) 51 | sim <- generate_model(sim, make_testmodel) 52 | sim <- simulate_from_model(sim, nsim = 2, index = 1:4) 53 | expect_error(run_method(sim, list(soft + refit)), "Could not find output of method") 54 | sim <- run_method(sim, soft) 55 | sim <- run_method(sim, soft + refit) 56 | sim <- run_method(sim, soft + mult_by_2) 57 | sim <- run_method(sim, soft + refit + mult_by_2) 58 | sim <- run_method(sim, 59 | list(soft, soft + refit, soft + refit + mult_by_2)) 60 | sim <- evaluate(sim, mse) 61 | expect_identical(list(soft, soft + refit) + mult_by_2, 62 | list(soft + mult_by_2, soft + refit + mult_by_2)) 63 | # we also want to make sure that these extended methods can be run in 64 | # parallel without error... 65 | skip_on_cran() # http://r-pkgs.had.co.nz/tests.html says risky to 66 | # test parallel code on CRAN 67 | sim <- run_method(sim, 68 | list(soft, soft + refit, soft + refit + mult_by_2), 69 | parallel = list(socket_names = 2)) 70 | unlink(dir, recursive = TRUE) 71 | }) 72 | 73 | -------------------------------------------------------------------------------- /tests/testthat/test-method.R: -------------------------------------------------------------------------------- 1 | options(simulator.verbose = FALSE) 2 | 3 | context("run_method") 4 | 5 | make_testmodel <- function() { 6 | return(new_model(name = "tm/n2", 7 | label = sprintf("Test model"), 8 | params = list(n = 2, x = runif(2)), 9 | simulate = function(x, n, nsim) { 10 | y <- list() 11 | for (i in seq(nsim)) 12 | y[[i]] <- x + rnorm(n) 13 | return(y) 14 | })) 15 | } 16 | 17 | my_method_no_list <- new_method(name = "mynl", 18 | label = "My method", 19 | method = function(model, draw) { 20 | return(mean(draw)) 21 | }) 22 | my_method <- new_method(name = "my", label = "My method", 23 | method = function(model, draw) { 24 | return(list(est = mean(draw), f = draw[1])) 25 | }) 26 | 27 | my_method_with_an_error <- new_method(name = "errorprone", 28 | label = "Error prone method", 29 | method = function(model, draw) { 30 | r <- runif(1) 31 | if (r > 0.99) 32 | stop(r, " is > 0.99!") 33 | }) 34 | 35 | remove_time_from_out <- function(out) { 36 | for (i in seq(length(out@out))) out@out[[i]] <- out@out$time <- 0 37 | } 38 | 39 | test_that("run_method handles multiple indices as expected", { 40 | dir <- file.path(tempdir(), "example") 41 | if (!dir.exists(dir)) dir.create(dir) 42 | mref <- generate_model(dir, make_testmodel) 43 | dref <- simulate_from_model(mref, 2, 1:3) 44 | out1 <- run_method(dref[1:2], my_method) 45 | out2 <- load_outputs(dir, "tm/n2", 1:2, "my") 46 | expect_identical(load(out1), out2) 47 | run_method(dref[2:3], my_method) 48 | out3 <- load_outputs(dir, "tm/n2", 2, "my") 49 | out4 <- load(dref[[2]]) 50 | expect_identical(remove_time_from_out(out3), 51 | remove_time_from_out(out2)) 52 | unlink(dir, recursive = TRUE) 53 | }) 54 | 55 | test_that("run_method handles multiple methods as expected", { 56 | dir <- file.path(tempdir(), "example") 57 | if (!dir.exists(dir)) dir.create(dir) 58 | mref <- generate_model(dir, make_testmodel) 59 | dref <- simulate_from_model(mref, nsim = 2, 1:2) 60 | run_method(dref, my_method) 61 | run_method(dref, my_method_no_list) 62 | out <- load_outputs(dir, "tm/n2", 2, "my") 63 | outt <- load_outputs(dir, "tm/n2", 2, "mynl") 64 | 65 | run_method(dref, list(my_method, my_method_no_list)) 66 | out2 <- load_outputs(dir, "tm/n2", 2, "my") 67 | outt2 <- load_outputs(dir, "tm/n2", 2, "mynl") 68 | expect_identical(remove_time_from_out(out), 69 | remove_time_from_out(out2)) 70 | expect_identical(remove_time_from_out(outt), 71 | remove_time_from_out(outt2)) 72 | unlink(dir, recursive = TRUE) 73 | }) 74 | 75 | test_that("run_method correctly directs user to error.", { 76 | dir <- file.path(tempdir(), "example") 77 | if (!dir.exists(dir)) dir.create(dir) 78 | sim <- new_simulation(name = "sim1", label = "test sim", dir = dir) %>% 79 | generate_model(make_testmodel) %>% 80 | simulate_from_model(index = 1:5, nsim = 10) 81 | # capture the error thrown if there's an error in a method... 82 | err_out <- tryCatch({ 83 | sim %>% run_method(list(my_method, my_method_with_an_error))}, 84 | error = function(e) return(e)) 85 | # the heart of the error message is the following: 86 | original_error_message <- regmatches(err_out$message, 87 | regexec("error message: (.*?)\n\nHint:", 88 | err_out$message))[[1]][2] 89 | # the full error message includes some suggested code to allow the user 90 | # to exactly reproduce the behavior that led to the error: 91 | suggested_code <- regmatches(err_out$message, 92 | regexec("object:\n\n(.*?)Note:", 93 | err_out$message))[[1]][2] 94 | # here we run that suggested code and check that the error message we 95 | # get exactly matches the original error message: 96 | met <- my_method_with_an_error 97 | expect_error(eval(parse(text = suggested_code))) 98 | err <- tryCatch({eval(parse(text = suggested_code))}, 99 | error = function(e) return(e)) 100 | expect_identical(original_error_message, err$message) 101 | unlink(dir, recursive = TRUE) 102 | }) 103 | 104 | -------------------------------------------------------------------------------- /tests/testthat/test-regex.R: -------------------------------------------------------------------------------- 1 | options(simulator.verbose = FALSE) 2 | 3 | context("component name checking") 4 | 5 | test_that("is_valid_component_name works", { 6 | expect_true(length(is_valid_component_name("a-2/1_b", "")) == 0) 7 | expect_true(length(is_valid_component_name("a.2/1b", "")) == 0) 8 | expect_true(length(is_valid_component_name("a.2.4-a.5/1b", "")) == 0) 9 | expect_true(length(is_valid_component_name("a,b", "")) == 1) 10 | expect_true(length(is_valid_component_name("a2/1%b", "")) == 1) 11 | expect_true(length(is_valid_component_name("/a2/1b", "")) == 1) 12 | expect_true(length(is_valid_component_name("a2/1b/", "")) == 1) 13 | expect_true(length(is_valid_component_name("a2//a", "")) == 1) 14 | expect_true(length(is_valid_component_name("a-_2/1b", "")) == 1) 15 | expect_true(length(is_valid_component_name("a2/_1b", "")) == 1) 16 | expect_true(length(is_valid_component_name("a2/_1b", "")) == 1) 17 | expect_true(length(is_valid_component_name("-a2/1b", "")) == 1) 18 | expect_true(length(is_valid_component_name("a2-/1b", "")) == 1) 19 | expect_true(length(is_valid_component_name("a2/1b_", "")) == 1) 20 | expect_true(length(is_valid_component_name("a2/1b/c/d/e/a", "")) == 0) 21 | expect_true(length(is_valid_component_name("a/a", "", 22 | allow_slash = FALSE)) == 1) 23 | }) 24 | -------------------------------------------------------------------------------- /tests/testthat/test-simulator.files.R: -------------------------------------------------------------------------------- 1 | options(simulator.verbose = FALSE) 2 | 3 | context("simulator.files") 4 | 5 | make_testmodel <- function() { 6 | return(new_model(name = "tm/n2", 7 | label = sprintf("Test model"), 8 | params = list(n = 2, x = runif(2)), 9 | simulate = function(x, n, nsim) { 10 | y <- list() 11 | for (i in seq(nsim)) 12 | y[[i]] <- x + rnorm(n) 13 | return(y) 14 | })) 15 | } 16 | 17 | my_method <- new_method(name = "my", 18 | label = "My method", 19 | method = function(model, draw) { 20 | return(list(est = mean(draw), 21 | f = draw[1])) 22 | }) 23 | 24 | squared_error <- new_metric(name = "se", 25 | label = "Squared Error", 26 | metric = function(model, out) { 27 | sum((out$est - model$x)^2) 28 | }) 29 | 30 | 31 | test_that("simulator.files is managed properly", { 32 | dir <- file.path(tempdir(), "example") 33 | sf <- "standard_name_for_files" 34 | sf2 <- "where_my_files_are_stored" 35 | options(simulator.files = sf) 36 | if (!dir.exists(dir)) dir.create(dir) 37 | # generate_model using sf 38 | mref <- generate_model(dir, make_testmodel) 39 | model <- load(mref) 40 | # now change to sf2 41 | options(simulator.files = sf2) 42 | model2 <- load(mref) 43 | expect_equal(model, model2) # not identical b/c function environments differ 44 | expect_output(show(mref), sf) # show(mref) should show simulator.files 45 | expect_error(simulate_from_model(mref, nsim = 1), "match getOption") 46 | options(simulator.files = sf) 47 | dref <- simulate_from_model(mref, nsim = 1) 48 | draws <- load(dref) 49 | options(simulator.files = sf2) 50 | draws2 <- load(dref) 51 | expect_identical(draws, draws2) 52 | expect_output(show(dref), sf) # show(dref) should show simulator.files 53 | expect_error(run_method(dref, my_method), "match getOption") 54 | options(simulator.files = sf) 55 | oref <- run_method(dref, my_method) 56 | options(simulator.files = sf2) 57 | expect_output(show(oref), sf) # show(oref) should show simulator.files 58 | expect_error(evaluate(oref, squared_error), "match getOption") 59 | options(simulator.files = sf) 60 | eref <- evaluate(oref, squared_error) 61 | unlink(dir, recursive = TRUE) 62 | options(simulator.files = "files") # restore default 63 | }) 64 | -------------------------------------------------------------------------------- /tests/testthat/test-table.R: -------------------------------------------------------------------------------- 1 | options(simulator.verbose = FALSE) 2 | 3 | context("tables") 4 | 5 | make_testmodel <- function(n) { 6 | return(new("Model", name = "tm", 7 | label = sprintf("Test model (n = %s)", n), 8 | params = list(n = n, mu = 2), 9 | simulate = function(mu, n, nsim) { 10 | y <- list() 11 | for (i in seq(nsim)) 12 | y[[i]] <- mu + rnorm(n) 13 | return(y) 14 | })) 15 | } 16 | 17 | his_method <- new("Method", 18 | name = "his", 19 | label = "His method", 20 | method = function(model, draw) { 21 | return(list(est = median(draw))) 22 | }) 23 | 24 | my_method <- new("Method", 25 | name = "my", 26 | label = "My method", 27 | method = function(model, draw) { 28 | return(list(est = mean(draw))) 29 | }) 30 | 31 | squared_error <- new("Metric", 32 | name = "sqerr", 33 | label = "Squared Error", 34 | metric = function(model, out) { 35 | sum((out$est - model$mu)^2) 36 | }) 37 | 38 | l1_error <- new("Metric", 39 | name = "l1", 40 | label = "L1 Error", 41 | metric = function(model, out) { 42 | sum(abs(out$est - model$mu)) 43 | }) 44 | 45 | test_that("can make a table", { 46 | dir <- file.path(tempdir(), "example") 47 | if (!dir.exists(dir)) dir.create(dir) 48 | mref <- generate_model(dir, make_testmodel, vary_along = "n", 49 | n = list(5, 100)) 50 | dref1 <- simulate_from_model(mref[[1]], nsim = 3, index = 1:3) 51 | dref2 <- simulate_from_model(mref[[2]], nsim = 3, index = 1:3) 52 | oref1 <- run_method(dref1, list(my_method, his_method)) 53 | oref2 <- run_method(dref2, list(my_method)) 54 | eref1 <- evaluate(oref1, list(squared_error, l1_error)) 55 | eref2 <- evaluate(oref2, list(l1_error)) 56 | evals1 <- load(eref1) 57 | evals2 <- load(eref2) 58 | evlist <- list(evals1, evals2) 59 | expect_error(tabulate_eval(evlist, "sqerr", method_names = "yours"), 60 | "not found in any evals") 61 | expect_error(tabulate_eval(evlist, "sqerr", method_names = "his", 62 | format_args = list(digits = 2)), NA) 63 | unlink(dir, recursive = TRUE) 64 | }) 65 | -------------------------------------------------------------------------------- /tests/testthat/test-utils.R: -------------------------------------------------------------------------------- 1 | options(simulator.verbose = FALSE) 2 | 3 | context("utils") 4 | 5 | test_that("memory_as_string works", { 6 | expect_identical(memory_as_string(10), "10 Bytes") 7 | expect_identical(memory_as_string(12.34), "12.34 Bytes") 8 | expect_identical(memory_as_string(123.45), "123.45 Bytes") 9 | expect_identical(memory_as_string(1234.56), "1.23 KB") 10 | expect_identical(memory_as_string(12345.6), "12.35 KB") 11 | expect_identical(memory_as_string(123456), "123.46 KB") 12 | expect_identical(memory_as_string(1234567), "1.23 MB") 13 | expect_identical(memory_as_string(12345678), "12.35 MB") 14 | expect_identical(memory_as_string(123456789), "123.46 MB") 15 | expect_identical(memory_as_string(1234567891), "1.23 GB") 16 | expect_identical(memory_as_string(12345678912), "12.35 GB") 17 | expect_identical(memory_as_string(123456789123), "123.46 GB") 18 | expect_identical(memory_as_string(1234567891234), "1.23 TB") 19 | }) 20 | 21 | test_that("get_relative_path works", { 22 | temp <- tempdir() 23 | dir <- file.path(temp, "example") 24 | sub <- file.path(dir, "sub") 25 | sub2 <- file.path(dir, "sub2") 26 | if (!dir.exists(dir)) dir.create(dir) 27 | if (!dir.exists(sub)) dir.create(sub) 28 | if (!dir.exists(sub2)) dir.create(sub2) 29 | expect_identical(get_relative_path(temp, dir), "example") 30 | expect_identical(get_relative_path(temp, sub), "example/sub") 31 | expect_identical(get_relative_path(dir, temp), "..") 32 | expect_identical(get_relative_path(sub, temp), "../..") 33 | expect_identical(get_relative_path(sub, sub2), "../sub2") 34 | unlink(dir, recursive = TRUE) 35 | wd <- strsplit(getwd(), split = "/")[[1]] 36 | lwd <- length(wd) 37 | expect_identical(get_relative_path("../..", "."), 38 | file.path(wd[lwd - 1], wd[lwd])) 39 | expect_identical(get_relative_path("../..", ".."), wd[lwd - 1]) 40 | expect_identical(get_relative_path("..", "../.."), "..") 41 | }) 42 | -------------------------------------------------------------------------------- /vignettes/elastic-net/eval_functions.R: -------------------------------------------------------------------------------- 1 | 2 | ## @knitr metrics 3 | 4 | sqr_err <- new_metric("sqr_err", "squared error", 5 | metric = function(model, out) { 6 | colMeans(as.matrix(out$beta - model$beta)^2) 7 | }) 8 | 9 | best_sqr_err <- new_metric("best_sqr_err", "best squared error", 10 | metric = function(model, out) { 11 | min(colMeans(as.matrix(out$beta - model$beta)^2)) 12 | }) 13 | 14 | nnz <- new_metric("nnz", "number of nonzeros", 15 | metric = function(model, out) { 16 | colSums(as.matrix(out$beta) != 0) 17 | }) 18 | -------------------------------------------------------------------------------- /vignettes/elastic-net/main.R: -------------------------------------------------------------------------------- 1 | # This is the main simulator file 2 | 3 | library(simulator) # this file was created under simulator version 0.0.0.9000 4 | 5 | source("model_functions.R") 6 | source("method_functions.R") 7 | source("eval_functions.R") 8 | 9 | ## @knitr init 10 | 11 | name_of_simulation <- "elastic-net" 12 | 13 | ## @knitr main 14 | 15 | sim <- new_simulation(name_of_simulation, "Elastic Nets") %>% 16 | generate_model(make_sparse_linear_model_with_corr_design, 17 | n = 100, p = 50, snr = 2, k = 10, 18 | rho = as.list(seq(0, 0.8, length = 6)), 19 | vary_along = "rho") %>% 20 | simulate_from_model(nsim = 3, index = 1:4) %>% 21 | run_method(list_of_elastic_nets, 22 | parallel = list(socket_names = 2, libraries = "glmnet")) %>% 23 | evaluate(list(sqr_err, nnz, best_sqr_err)) 24 | 25 | ## @knitr maincv 26 | 27 | sim_cv <- sim %>% subset_simulation(methods = "") %>% 28 | rename("elastic-net-cv") %>% 29 | relabel("Elastic Nets with CV") %>% 30 | run_method(list_of_elastic_nets + cv, 31 | parallel = list(socket_names = 2, libraries = "glmnet")) %>% 32 | evaluate(list(sqr_err, nnz)) 33 | 34 | ## @knitr plots 35 | 36 | ## @knitr plotscv 37 | 38 | plot_eval_by(sim_cv, "sqr_err", varying = "rho", include_zero = TRUE) 39 | 40 | ## @knitr tables 41 | 42 | -------------------------------------------------------------------------------- /vignettes/elastic-net/method_functions.R: -------------------------------------------------------------------------------- 1 | 2 | ## @knitr methods 3 | 4 | library(glmnet) 5 | make_elastic_net <- function(alpha) { 6 | new_method(name = sprintf("en%s", alpha), 7 | label = sprintf("EN(a = %s)", alpha), 8 | settings = list(alpha = alpha, nlam = 50), 9 | method = function(model, draw, alpha, nlam, lambda = NULL) { 10 | if (is.null(lambda)) 11 | fit <- glmnet(x = model$x, y = draw, alpha = alpha, 12 | nlambda = nlam, intercept = FALSE) 13 | else 14 | fit <- glmnet(x = model$x, y = draw, alpha = alpha, 15 | lambda = lambda, intercept = FALSE) 16 | list(beta = fit$beta, yhat = model$x %*% fit$beta, 17 | lambda = fit$lambda, df = fit$df, alpha = alpha) 18 | }) 19 | } 20 | 21 | list_of_elastic_nets <- sapply(c(0, 0.5, 1), make_elastic_net) 22 | 23 | 24 | ## @knitr cv 25 | 26 | #' Make folds for cross validation 27 | #' 28 | #' Divides the indices \code{1:n} into \code{nfolds} random folds of about the same size. 29 | #' 30 | #' @param n sample size 31 | #' @param nfolds number of folds 32 | make_folds <- function(n, nfolds) { 33 | nn <- round(n / nfolds) 34 | sizes <- rep(nn, nfolds) 35 | sizes[nfolds] <- sizes[nfolds] + n - nn * nfolds 36 | b <- c(0, cumsum(sizes)) 37 | ii <- sample(n) 38 | folds <- list() 39 | for (i in seq(nfolds)) 40 | folds[[i]] <- ii[seq(b[i] + 1, b[i + 1])] 41 | folds 42 | } 43 | 44 | cv <- new_method_extension("cv", "cross validated", 45 | method_extension = function(model, draw, out, 46 | base_method) { 47 | nfolds <- 5 48 | alpha <- base_method@settings$alpha 49 | err <- matrix(NA, ncol(out$beta), nfolds) 50 | ii <- make_folds(model$n, nfolds) 51 | for (i in seq_along(ii)) { 52 | train <- model 53 | train@params$x <- model@params$x[-ii[[i]], ] 54 | train@params$n <- model@params$x[-ii[[i]], ] 55 | train_draw <- draw[-ii[[i]]] 56 | 57 | test <- model 58 | test@params$x <- model@params$x[ii[[i]], ] 59 | test@params$n <- model@params$x[ii[[i]], ] 60 | test_draw <- draw[ii[[i]]] 61 | fit <- base_method@method(model = train, 62 | draw = train_draw, 63 | alpha = alpha, 64 | lambda = out$lambda) 65 | yhat <- test$x %*% fit$beta 66 | ll <- seq(ncol(yhat)) 67 | err[ll, i] <- colMeans((yhat - test_draw)^2) 68 | } 69 | m <- rowMeans(err) 70 | se <- apply(err, 1, sd) / sqrt(nfolds) 71 | imin <- which.min(m) 72 | ioneserule <- max(which(m <= m[imin] + se[imin])) 73 | list(err = err, m = m, se = se, imin = imin, 74 | ioneserule = ioneserule, 75 | beta = out$beta[, imin], 76 | yhat = model$x %*% out$beta[, imin], 77 | alpha = alpha) 78 | }) 79 | 80 | -------------------------------------------------------------------------------- /vignettes/elastic-net/model_functions.R: -------------------------------------------------------------------------------- 1 | 2 | ## @knitr models 3 | 4 | library(mvtnorm) 5 | make_sparse_linear_model_with_corr_design <- function(n, p, k, snr, rho) { 6 | sig <- matrix(rho, p, p) 7 | diag(sig) <- 1 8 | x <- rmvnorm(n, sigma = sig) 9 | beta <- rep(c(1, 0), c(k, p - k)) 10 | mu <- as.numeric(x %*% beta) 11 | sigma <- sqrt(sum(mu^2) / (n * snr)) # taking snr = ||mu||^2 / (n * sigma^2) 12 | new_model(name = "slm", label = sprintf("rho = %s", rho), 13 | params = list(x = x, beta = beta, mu = mu, sigma = sigma, n = n, 14 | p = p, k = k), 15 | simulate = function(mu, sigma, nsim) { 16 | y <- mu + sigma * matrix(rnorm(nsim * n), n, nsim) 17 | return(split(y, col(y))) # make each col its own list element 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /vignettes/false-discovery-rate/eval_functions.R: -------------------------------------------------------------------------------- 1 | ## @knitr metrics 2 | 3 | fdp <- new_metric(name = "fdp", 4 | label = "false discovery proportion", 5 | metric = function(model, out) { 6 | fp <- setdiff(out$rejected, model$nonnull) 7 | nd <- max(length(out$rejected), 1) 8 | return(length(fp) / nd) 9 | }) 10 | 11 | nd <- new_metric(name = "nd", 12 | label = "number of discoveries", 13 | metric = function(model, out) length(out$rejected)) 14 | -------------------------------------------------------------------------------- /vignettes/false-discovery-rate/main.R: -------------------------------------------------------------------------------- 1 | # This is the main simulator file 2 | 3 | library(simulator) # this file was created under simulator version 0.0.0.9000 4 | 5 | source("model_functions.R") 6 | source("method_functions.R") 7 | source("eval_functions.R") 8 | 9 | ## @knitr init 10 | 11 | name_of_simulation <- "fdr" 12 | 13 | ## @knitr main 14 | 15 | sim <- new_simulation(name = name_of_simulation, 16 | label = "False Discovery Rate") %>% 17 | generate_model(make_correlated_pvalues, seed = 123, 18 | n = 20, 19 | pi0 = list(0.8, 1), 20 | rho = list(-0.01, 0, 0.1, 0.9), 21 | vary_along = c("pi0", "rho")) %>% 22 | simulate_from_model(nsim = 25, index = 1:4) %>% 23 | run_method(bh_methods, parallel = list(socket_names = 2)) %>% 24 | evaluate(list(fdp, nd)) 25 | 26 | ## @knitr main2 27 | 28 | sim <- sim %>% 29 | simulate_from_model(nsim = 25, index = 5:8) %>% 30 | run_method(bh_methods, parallel = list(socket_names = 2)) %>% 31 | evaluate(list(fdp, nd)) 32 | 33 | ## @knitr main3 34 | 35 | sim2 <- subset_simulation(sim, pi0 == 1 & rho == -0.01) %>% 36 | rename("fdr-negdep") %>% 37 | relabel("BH Procedure under negative dependence") %>% 38 | simulate_from_model(nsim = 25, index = 9:20) %>% 39 | run_method(bh_methods, parallel = list(socket_names = 2)) %>% 40 | evaluate(list(fdp, nd)) 41 | 42 | -------------------------------------------------------------------------------- /vignettes/false-discovery-rate/method_functions.R: -------------------------------------------------------------------------------- 1 | ## @knitr methods 2 | make_bh <- function(q) { 3 | # q is the desired level of control for the FDR 4 | new_method(name = paste0("bh", q), 5 | label = sprintf("BH (q = %s)", q), 6 | settings = list(q = q), 7 | method = function(model, draw, q) { 8 | p <- sort(draw) 9 | cutline <- seq(model$n) * q / model$n 10 | threshold <- max(p[p <= cutline], 0) 11 | list(rejected = which(draw <= threshold)) 12 | }) 13 | } 14 | 15 | qvalues <- c(0.05, 0.1, 0.2) 16 | bh_methods <- sapply(qvalues, make_bh) 17 | -------------------------------------------------------------------------------- /vignettes/false-discovery-rate/model_functions.R: -------------------------------------------------------------------------------- 1 | ## @knitr models 2 | library(mvtnorm) 3 | make_correlated_pvalues <- function(n, pi0, rho) { 4 | # Gaussian copula model... 5 | # 6 | # n pvalues, the first n*pi0 of which are null, coming from a multivariate 7 | # normal with all correlations rho. 8 | sigma <- matrix(rho, n, n) 9 | diag(sigma) <- 1 10 | n0 <- round(n * pi0) 11 | delta <- 2 # size of signal 12 | mu <- rep(c(0, delta), c(n0, n - n0)) # n0 are null 13 | new_model(name = "correlated-pvalues", 14 | label = sprintf("pi0 = %s, rho = %s", pi0, rho), 15 | params = list(n = n, rho = rho, sigma = sigma, 16 | pi0 = pi0, mu = mu, delta = delta, 17 | nonnull = which(mu != 0)), 18 | simulate = function(n, mu, sigma, nsim) { 19 | # this function must return a list of length nsim 20 | x <- rmvnorm(nsim, mean = mu, sigma = sigma) 21 | pvals <- 1 - pnorm(x) 22 | return(split(pvals, row(pvals))) # make each row its own list element 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /vignettes/james-stein/eval_functions.R: -------------------------------------------------------------------------------- 1 | ## @knitr metrics 2 | 3 | sqr_err <- new_metric(name = "sqrerr", label = "Squared Error Loss", 4 | metric = function(model, out) { 5 | mean((out$est - model$theta)^2) 6 | }) 7 | -------------------------------------------------------------------------------- /vignettes/james-stein/main.R: -------------------------------------------------------------------------------- 1 | # This is the main simulator file 2 | 3 | library(simulator) # this file was created under simulator version 0.0.0.9000 4 | 5 | source("model_functions.R") 6 | source("method_functions.R") 7 | source("eval_functions.R") 8 | 9 | ## @knitr init 10 | 11 | ## @knitr main1 12 | 13 | sim1 <- new_simulation(name = "js-v-mle", 14 | label = "Investigating the James-Stein Estimator") %>% 15 | generate_model(make_normal_model, theta_norm = 1, p = list(2, 6), 16 | vary_along = "p", seed = 123) %>% 17 | simulate_from_model(nsim = 20) %>% 18 | run_method(list(js, mle)) %>% 19 | evaluate(sqr_err) 20 | 21 | ## @knitr main2 22 | 23 | sim2 <- new_simulation(name = "js-v-mle2", 24 | label = "Investigating James-Stein Estimator") %>% 25 | generate_model(make_normal_model, vary_along = "p", 26 | theta_norm = 1, p = as.list(seq(1, 30, by = 5))) %>% 27 | simulate_from_model(nsim = 20) %>% 28 | run_method(list(js, mle)) %>% 29 | evaluate(sqr_err) 30 | 31 | ## @knitr main3 32 | 33 | sim3 <- new_simulation(name = "js-v-mle3", 34 | label = "Investigating the James-Stein Estimator") %>% 35 | generate_model(make_normal_model, vary_along = c("p", "theta_norm"), 36 | theta_norm = as.list(round(seq(0, 5, length = 10), 2)), 37 | p = as.list(seq(1, 30, by = 10))) %>% 38 | simulate_from_model(nsim = 20) %>% 39 | run_method(list(js, mle)) %>% 40 | evaluate(sqr_err) 41 | -------------------------------------------------------------------------------- /vignettes/james-stein/method_functions.R: -------------------------------------------------------------------------------- 1 | ## @knitr methods 2 | 3 | mle <- new_method(name = "mle", label = "MLE", 4 | method = function(model, draw) return(list(est = draw))) 5 | 6 | js <- new_method(name = "jse", label = "James-Stein", 7 | method = function(model, draw) { 8 | l2 <- sum(draw^2) 9 | return(list(est = (1 - (model$p - 2) / l2) * draw)) 10 | }) 11 | -------------------------------------------------------------------------------- /vignettes/james-stein/model_functions.R: -------------------------------------------------------------------------------- 1 | ## @knitr models 2 | 3 | make_normal_model <- function(theta_norm, p) { 4 | new_model(name = "norm", 5 | label = sprintf("p = %s, theta_norm = %s", p, theta_norm), 6 | params = list(theta_norm = theta_norm, p = p, 7 | theta = c(theta_norm, rep(0, p - 1))), 8 | simulate = function(theta, p, nsim) { 9 | Y <- theta + matrix(rnorm(nsim * p), p, nsim) 10 | return(split(Y, col(Y))) # make each col its own list element 11 | }) 12 | } 13 | -------------------------------------------------------------------------------- /vignettes/lasso/eval_functions.R: -------------------------------------------------------------------------------- 1 | 2 | ## @knitr metrics 3 | 4 | sqrerr <- new_metric("sqrerr", "squared error", 5 | metric = function(model, out) { 6 | colMeans(as.matrix(out$beta - model$beta)^2) 7 | }) 8 | 9 | best_sqrerr <- new_metric("best_sqrerr", "best squared error", 10 | metric = function(model, out) { 11 | min(colMeans(as.matrix(out$beta - model$beta)^2)) 12 | }) 13 | 14 | nnz <- new_metric("nnz", "number of nonzeros", 15 | metric = function(model, out) { 16 | colSums(as.matrix(out$beta) != 0) 17 | }) 18 | 19 | df <- new_metric("df", "degrees of freedom", 20 | metric = function(model, out) out$df) 21 | -------------------------------------------------------------------------------- /vignettes/lasso/main.R: -------------------------------------------------------------------------------- 1 | 2 | ## @knitr main1 3 | 4 | sim <- new_simulation("bet-on-sparsity", "Bet on sparsity") %>% 5 | generate_model(make_sparse_linear_model, n = 200, p = 500, snr = 2, 6 | k = as.list(seq(5, 80, by = 5)), 7 | vary_along = "k") %>% 8 | simulate_from_model(nsim = 2, index = 1:2) %>% 9 | run_method(list(lasso, ridge), 10 | parallel = list(socket_names = 2, libraries = "glmnet")) %>% 11 | evaluate(list(sqrerr, nnz, df, best_sqrerr)) 12 | 13 | ## @knitr main2 14 | 15 | sim2 <- sim %>% 16 | subset_simulation(methods = "lasso") %>% 17 | rename("relaxing-the-lasso") %>% 18 | relabel("Effect of relaxing lasso") %>% 19 | run_method(methods = lasso + refit) %>% 20 | evaluate(list(sqrerr, nnz, df, best_sqrerr)) 21 | 22 | ## @knitr main3 23 | 24 | sim3 <- sim %>% subset_simulation(methods = "") %>% 25 | rename("bet-on-sparsity-cv") %>% 26 | relabel("Bet on sparsity (with cross validation)") %>% 27 | run_method(list(lasso + cv, ridge + cv)) %>% 28 | evaluate(list(sqrerr, nnz)) 29 | -------------------------------------------------------------------------------- /vignettes/lasso/model_functions.R: -------------------------------------------------------------------------------- 1 | 2 | ## @knitr models 3 | 4 | make_sparse_linear_model <- function(n, p, k, snr) { 5 | x <- matrix(rnorm(n * p), n, p) 6 | beta <- rep(c(1, 0), c(k, p - k)) 7 | mu <- as.numeric(x %*% beta) 8 | sigma <- sqrt(sum(mu^2) / (n * snr)) # taking snr = ||mu||^2 / (n * sigma^2) 9 | new_model(name = "slm", label = sprintf("n = %s, p = %s, k = %s", n, p, k), 10 | params = list(x = x, beta = beta, mu = mu, sigma = sigma, n = n, 11 | p = p, k = k), 12 | simulate = function(mu, sigma, nsim) { 13 | return(lapply(1:nsim, function(i) mu + sigma * rnorm(n))) 14 | }) 15 | } 16 | -------------------------------------------------------------------------------- /vignettes/simulator.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Getting Started with the Simulator" 3 | output: rmarkdown::html_vignette 4 | vignette: > 5 | %\VignetteEngine{knitr::rmarkdown} 6 | %\VignetteIndexEntry{Getting Started with the Simulator} 7 | \usepackage[utf8]{inputenc} 8 | --- 9 | 10 | This brief vignette describes how to get started with the `simulator`. 11 | 12 | ## Starting from a template 13 | 14 | After installing the package, open R and type. 15 | ```{r} 16 | library(simulator) 17 | dir <- "./sims" 18 | create(dir) 19 | ``` 20 | 21 | Choose `dir` to be the path of a directory (that does not yet exist) where you want your simulation code and files to be stored. In practice, `"./sims"` would be a standard choice, where `"."` refers to a directory containing files relevant to your current project. 22 | 23 | The `create` command generates a skeleton of a simulation.^[This function was inspired by the `create` function in `devtools`, which creates the skeleton of an R package.] A look at the newly created directory shows that several files have been created. 24 | 25 | ```{r, eval=FALSE} 26 | setwd(dir) 27 | list.files() 28 | ``` 29 | 30 | ```{r, echo=FALSE} 31 | list.files(dir) 32 | ``` 33 | 34 | This is the template of a basic simulation. 35 | 36 | - In `model_functions.R`, write code that defines the models under which you wish to simulate. 37 | - In `method_functions.R`, add code for methods that you wish to compare in your simulation (note that by using `source` and `library`, you can keep `method_functions.R` short and to the point, focusing on calling `new_method` rather than putting the actual heart of algorithms in that file). 38 | - In `eval_functions.R`, use `new_metric` to define the ways in which your methods will be evaluated. 39 | - The file `main.R` contains the main entry point to the simulation. Running the code in this file determines which models/methods/metrics are computed, etc. 40 | - Finally, the file `writeup.Rmd` shows how all results can be presented in as a report. This document pulls all code from the `.R` files mentioned above, so that as `main.R` and other files develop, the report will remain up to date. To create an `html` file report, run the following command in R (which requires installing the package `rmarkdown`). 41 | 42 | ```{r, eval = FALSE} 43 | rmarkdown::render("writeup.Rmd", "html_document") 44 | ``` 45 | 46 | Or if one is using RStudio, one can simply press the `Knit HTML` button. 47 | 48 | ## Typical workflow 49 | 50 | On a typical project, one starts by defining a model in `model_functions.R`, one or two methods in `method_functions.R`, and a few metrics in `eval_functions.R`, and then one runs the code in `main.R`. After looking at some of the results, one might add an additional model or method or metric. One then returns to `main.R`, adds some additional lines specifying that the additional components should be run as well and looks at some more results. 51 | 52 | The simplest way to look at results is by using the plot functions `plot_eval`, `plot_evals` and `plot_evals_by`. In situations where you wish to investigate results more deeply than just looking at aggregated plots, one can use the functions `model`, `draws`, `output`, and `evals` to get at all objects generated through the course of the simulation. 53 | 54 | ## Next steps 55 | 56 | The best way to get a sense of how to use the simulator is to look at examples. There are several vignettes that demonstrate how the simulator can be used to conduct simulations for some of the most famous statistical methods. 57 | 58 | 1) [Lasso](lasso.html) vignette: Explains basics, including the magrittr pipe and making plots and tables. Also demonstrates some more advanced features such as writing method extensions (such as refitting the result of the lasso or performing cross-validation). 59 | 2) [James-Stein](james-stein.html) vignette: Shows how to step into specific parts of the simulation for troubleshooting your code. 60 | 3) [Elastic net](elastic-net.html) vignette: Shows how we can work with a sequence of methods that are identical except for a parameter that varies 61 | 4) [Benjamini-Hochberg](fdr.html) vignette: Shows how we can load a preexisting simulation and add more random draws without having to rerun anything. It also shows how one can have multiple simulation objects that point to overlapping sets of results. 62 | 63 | ```{r, echo = FALSE} 64 | # delete all files created in running this vignette 65 | unlink(dir, recursive = TRUE) 66 | ``` 67 | --------------------------------------------------------------------------------