├── .Rbuildignore ├── .gitignore ├── DESCRIPTION ├── NAMESPACE ├── R ├── conf_to_dt.R ├── configure_task.R ├── cron_funs.R ├── launcher.R ├── module_configure_task.R ├── module_tasks_overview.R ├── run_order.R ├── run_task.R └── utils.R ├── README.md ├── inst ├── article │ ├── article.html │ └── article.md ├── demo_app │ ├── global.R │ ├── server.R │ ├── ui.R │ └── www │ │ ├── css │ │ └── custom.css │ │ ├── figures │ │ ├── conf_file.PNG │ │ ├── custom_cron.PNG │ │ ├── default_cron.PNG │ │ ├── demo_app_conf_task.PNG │ │ ├── demo_app_view_task_1.PNG │ │ ├── demo_app_view_task_2.PNG │ │ ├── launch_task_shiny.PNG │ │ ├── launch_task_shiny_2.PNG │ │ ├── launch_task_shiny_code.PNG │ │ ├── see_tasks_shiny.PNG │ │ ├── see_tasks_shiny_2.PNG │ │ └── see_tasks_shiny_code.PNG │ │ ├── img │ │ ├── github.png │ │ ├── img-datastorm-logo-white.png │ │ ├── logoDS.png │ │ └── rpivot.PNG │ │ └── script │ │ ├── global.md │ │ ├── intro.md │ │ ├── server.md │ │ └── ui.md ├── demo_app_conc │ ├── global.R │ ├── server.R │ ├── ui.R │ └── www │ │ ├── css │ │ └── custom.css │ │ ├── figures │ │ ├── conf_file.PNG │ │ ├── custom_cron.PNG │ │ ├── default_cron.PNG │ │ ├── demo_app_conf_task.PNG │ │ ├── demo_app_view_task_1.PNG │ │ ├── demo_app_view_task_2.PNG │ │ ├── launch_task_shiny.PNG │ │ ├── launch_task_shiny_2.PNG │ │ ├── launch_task_shiny_code.PNG │ │ ├── see_tasks_shiny.PNG │ │ ├── see_tasks_shiny_2.PNG │ │ └── see_tasks_shiny_code.PNG │ │ ├── img │ │ ├── github.png │ │ ├── img-datastorm-logo-white.png │ │ ├── logoDS.png │ │ └── rpivot.PNG │ │ └── script │ │ ├── global.md │ │ ├── intro.md │ │ ├── server.md │ │ └── ui.md ├── ex_fun │ ├── sb_fun_demo_app_python.R │ ├── sb_fun_demo_app_python.py │ ├── sb_fun_ex.R │ └── sb_fun_ex_demo_app.R └── scripts │ └── test.R ├── man ├── configuration_info.Rd ├── configure_task.Rd ├── input_btns.Rd ├── launcher.Rd ├── module_configure_task.Rd ├── module_tasks_overview.Rd ├── run_order.Rd ├── run_task.Rd └── scheduler_shinybatch.Rd ├── tests ├── testthat.R └── testthat │ ├── test_configure_task.R │ ├── test_launcher.R │ ├── test_run_order.R │ └── test_run_task.R └── vignettes ├── .gitignore └── lancement_python.Rmd /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | ^doc$ 4 | ^Meta$ 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | *\.Rproj 6 | inst/doc 7 | doc 8 | Meta 9 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: shinybatch 2 | Title: Launch Tasks in Batch 3 | Version: 0.3.2 4 | Authors@R: c( 5 | person("Benoit", "Thieurmel", role = c("aut", "cre"), 6 | email = "bthieurmel@gmail.com"), 7 | person(given = "Thibaut", family = "Dubois", 8 | email = "thibaut.dubois@datastorm.fr", role = c("aut", "ctb")) 9 | ) 10 | Description: Provides two modules to configure some R task, launch in batch, and retrieve result from shiny. Can be used both in windows system using Windows task scheduler, and on unix-alike system using 'crontab'. 11 | Depends: R (>= 3.5.0) 12 | License: GPL-3 13 | Encoding: UTF-8 14 | RoxygenNote: 7.2.1 15 | Imports: 16 | data.table, 17 | DT, 18 | shiny, 19 | yaml, 20 | futile.logger, 21 | htmltools 22 | Suggests: 23 | testthat, 24 | cronR, 25 | taskscheduleR, 26 | knitr, 27 | rmarkdown 28 | VignetteBuilder: knitr 29 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | export(conf_to_dt) 4 | export(configure_task) 5 | export(configure_task_server) 6 | export(dir_conf_to_dt) 7 | export(launcher) 8 | export(run_order) 9 | export(run_task) 10 | export(scheduler_add) 11 | export(scheduler_exist) 12 | export(scheduler_init) 13 | export(scheduler_ls) 14 | export(scheduler_remove) 15 | export(tasks_overview_UI) 16 | export(tasks_overview_server) 17 | import(data.table) 18 | import(futile.logger) 19 | import(htmltools) 20 | import(shiny) 21 | import(yaml) 22 | importFrom(DT,"%>%") 23 | importFrom(DT,DTOutput) 24 | importFrom(DT,datatable) 25 | importFrom(DT,formatStyle) 26 | importFrom(DT,renderDT) 27 | importFrom(utils,packageVersion) 28 | importFrom(utils,read.delim2) 29 | -------------------------------------------------------------------------------- /R/conf_to_dt.R: -------------------------------------------------------------------------------- 1 | #' Convert a list of task configurations into two data.tables of global and individual features. 2 | #' 3 | #' @param dir_path \code{character}. Path to the directory with tasks. 4 | #' @param confs \code{list of list}. List of conf list(s) from yaml file(s). 5 | #' @param allowed_run_info_cols \code{characteror or boolean} (c("date_creation", "date_start", "date_end", "priority", "status")). Run info elements to be kept. 6 | #' @param allow_descr \code{boolean or character} (TRUE). Either a boolean specifying whether or not to keep descr elements, or column names. 7 | #' @param allowed_function_cols \code{character or boolean} (c("names", "path")). Function elements to be kept. 8 | #' @param allow_args \code{boolean or character} (TRUE). Either a boolean specifying whether or not to keep args elements, or column names. 9 | #' 10 | #' @return a list of two tables 'tbl_global' and 'tbl_idv': 11 | #' \itemize{ 12 | #' \item{tbl_global} {contains the global features of the configurations (.$run_info and .$descriptive)} 13 | #' \item{tbl_idv} {contains the individual features of the configurations (.$function and .$args)} 14 | #' } 15 | #' @export 16 | #' 17 | #' @import data.table 18 | #' 19 | #' @examples 20 | #' 21 | #' \donttest{ 22 | #' 23 | #' dir_conf <- paste0(tempdir(), "/conf") 24 | #' dir.create(dir_conf, recursive = TRUE) 25 | #' 26 | #' # ex fun 27 | #' fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 28 | #' fun_name = "sb_fun_ex" 29 | #' 30 | #' # create 2 confs 31 | #' conf_1 <- configure_task(dir_path = dir_conf, 32 | #' conf_descr = list(title_1 = "my_title_1", 33 | #' description_1 = "my_descr_1"), 34 | #' fun_path = fun_path, 35 | #' fun_name = fun_name, 36 | #' fun_args = list(x = 0, 37 | #' y = 0:4, 38 | #' z = iris), 39 | #' priority = 1) 40 | #' conf_2 <- configure_task(dir_path = dir_conf, 41 | #' conf_descr = list(title_2 = "my_title_2", 42 | #' description_2 = "my_descr_2"), 43 | #' fun_path = fun_path, 44 | #' fun_name = fun_name, 45 | #' fun_args = list(x = 1, 46 | #' y = 0:4, 47 | #' z = iris), 48 | #' priority = 2) 49 | #' 50 | #' # retrieve information about all tasks in main directory 51 | #' dir_conf_to_dt(dir_conf, allowed_run_info_cols = FALSE) 52 | #' 53 | #' dir_conf_to_dt(dir_conf, 54 | #' allow_descr = FALSE, 55 | #' allow_args = FALSE) 56 | #' 57 | #' dir_conf_to_dt(dir_conf, 58 | #' allowed_run_info_cols = c("status", "date_creation"), 59 | #' allowed_function_cols = c("path")) 60 | #' 61 | #' dir_conf_to_dt(dir_conf, 62 | #' allowed_run_info_cols = NULL, 63 | #' allowed_function_cols = NULL) 64 | #' 65 | #' dir_conf_to_dt(dir_conf, 66 | #' allowed_run_info_cols = "", 67 | #' allowed_function_cols = "", 68 | #' allow_descr = FALSE, 69 | #' allow_args = FALSE) 70 | #' 71 | #' # or just on some tasks ? 72 | #' info_conf_1 <- conf_1 73 | #' info_conf_2 <- yaml::read_yaml(file.path(conf_2$dir, "conf.yml")) 74 | #' 75 | #' conf_to_dt(list(info_conf_1, info_conf_2)) 76 | #' 77 | #' } 78 | #' 79 | #' @rdname configuration_info 80 | conf_to_dt <- function(confs, 81 | allowed_run_info_cols = c("date_creation", "date_start", "date_end", "priority", "status"), 82 | allow_descr = TRUE, 83 | allowed_function_cols = c("path", "name"), 84 | allow_args = TRUE) { 85 | 86 | # checks 87 | if (is.null(allowed_run_info_cols)) { 88 | allowed_run_info_cols <- c("date_creation", "date_start", "date_end", "priority", "status") 89 | } else if(is.logical(allowed_run_info_cols)) { 90 | if(allowed_run_info_cols){ 91 | allowed_run_info_cols <- c("date_creation", "date_start", "date_end", "priority", "status") 92 | } else { 93 | allowed_run_info_cols <- "" 94 | } 95 | } 96 | if (is.null(allowed_function_cols)) { 97 | allowed_function_cols <- c("path", "name") 98 | } else if(is.logical(allowed_function_cols)) { 99 | if(allowed_function_cols){ 100 | allowed_function_cols <- c("path", "name") 101 | } else { 102 | allowed_function_cols <- "" 103 | } 104 | } 105 | if (! is.character(allowed_run_info_cols)) { 106 | stop("'allowed_run_info_cols' must be of class .") 107 | } 108 | if (! is.character(allowed_function_cols)) { 109 | stop("'allowed_function_cols' must be of class .") 110 | } 111 | 112 | # create tables from lists 113 | tbl_global <- list() 114 | tbls_idv <- list() 115 | 116 | if (length(confs) > 0) { 117 | for (n_conf in 1:length(confs)) { 118 | cur_conf <- confs[[n_conf]] 119 | 120 | # get selected columns 121 | global_cols <- c( 122 | if (is.logical(allow_descr)) { 123 | if (allow_descr) { 124 | names(cur_conf$descriptive) 125 | } else {""} 126 | } else { 127 | allow_descr 128 | }, 129 | allowed_run_info_cols 130 | ) 131 | 132 | idv_cols <- c( 133 | allowed_function_cols, 134 | if (is.logical(allow_args)) { 135 | if (allow_args) { 136 | names(cur_conf$args) 137 | } else {""} 138 | } else {allow_args} 139 | ) 140 | 141 | tbl_global[[n_conf]] <- data.table::as.data.table(c(cur_conf["dir"], cur_conf$run_info, cur_conf$descriptive) 142 | )[, c("dir", intersect(global_cols, c(names(cur_conf$run_info), names(cur_conf$descriptive)))), with = FALSE] 143 | 144 | tbls_idv[[n_conf]] <- data.table::as.data.table(c(cur_conf[["function"]], cur_conf$args) 145 | )[, intersect(idv_cols, c(names(cur_conf[["function"]]), names(cur_conf$args))), with = FALSE] 146 | 147 | } 148 | } 149 | 150 | # rbind global table 151 | tbl_global <- data.table::rbindlist(tbl_global, fill = TRUE) 152 | 153 | # sort by decreasing priority and increasing date 154 | if ("date_creation" %in% names(tbl_global)) { 155 | tbls_idv <- tbls_idv[order(tbl_global[["date_creation"]], decreasing = FALSE)] 156 | tbl_global <- tbl_global[order(get("date_creation"), decreasing = FALSE)] 157 | } 158 | if ("priority" %in% names(tbl_global)) { 159 | tbls_idv <- tbls_idv[order(tbl_global[["priority"]], decreasing = TRUE)] 160 | tbl_global <- tbl_global[order(get("priority"), decreasing = TRUE)] 161 | } 162 | 163 | colnames(tbl_global) <- gsub("^date_creation$", "initialization_date", colnames(tbl_global)) 164 | colnames(tbl_global) <- gsub("^date_start$", "starting_date", colnames(tbl_global)) 165 | colnames(tbl_global) <- gsub("^date_end$", "ending_date", colnames(tbl_global)) 166 | 167 | return(list("tbl_global" = tbl_global, "tbls_idv" = tbls_idv)) 168 | } 169 | 170 | #' @rdname configuration_info 171 | #' @export 172 | dir_conf_to_dt <- function(dir_path, 173 | allowed_run_info_cols = c("date_creation", "date_start", "date_end", "priority", "status"), 174 | allow_descr = TRUE, 175 | allowed_function_cols = c("path", "name"), 176 | allow_args = TRUE){ 177 | 178 | # checks 179 | if (! is.character(dir_path)) { 180 | stop("'dir_path' must be of class .") 181 | } 182 | if (! dir.exists(dir_path)) { 183 | stop("'dir_path' directory doesn't exist (", dir_path, ").") 184 | } 185 | 186 | confs <- lapply(list.dirs(dir_path, full.names = TRUE, recursive = FALSE), function(x) { 187 | yaml::read_yaml(paste0(x, "/conf.yml")) 188 | }) 189 | 190 | if(length(confs) == 0){ 191 | return(invisible(NULL)) 192 | } else { 193 | conf_to_dt(confs = confs, 194 | allowed_run_info_cols = allowed_run_info_cols, 195 | allow_descr = allow_descr, 196 | allowed_function_cols = allowed_function_cols, 197 | allow_args = allow_args) 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /R/configure_task.R: -------------------------------------------------------------------------------- 1 | #' Initialize a configuration file for a future task 2 | #' 3 | #' @param dir_path \code{character}. Tasks location (parent directory). 4 | #' @param fun_path \code{character}. Path to the script of the function. 5 | #' @param fun_name \code{character}. Name of the function in fun_path script. 6 | #' @param conf_descr \code{named list} (NULL). Optional description fields. 7 | #' @param fun_args \code{named list} (NULL). Args of the function, must all be named. 8 | #' @param priority \code{numeric} (0L). Number used to define which task should be launched first using \code{\link[shinybatch]{launcher}} 9 | #' @param compress \code{logical or character} (TRUE). Either a logical specifying whether or not to use "gzip" compression, or one of "gzip", "bzip2" or "xz" 10 | #' to indicate the type of compression to be used for function result 11 | #' @param call. \code{logical} (TRUE) logical, indicating if the call should become part of the error message (in log file) 12 | #' 13 | #' @return a list containing the conf fields. Attribute 'path' of the result contains the path to the conf directory. 14 | #' The arg field contains either the location of the argument (in "dir_path/inputs/arg_name.RDS") or 15 | #' the argument itself if it is of length 1. 16 | #' 17 | #' @export 18 | #' 19 | #' @import yaml 20 | #' 21 | #' @examples 22 | #' 23 | #' \donttest{ 24 | #' 25 | #' 26 | #' # create temporary directory 27 | #' dir <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 28 | #' dir.create(dir, recursive = TRUE) 29 | #' 30 | #' # create and save conf 31 | #' conf <- configure_task(dir_path = dir, 32 | #' conf_descr = list(title = "my_title", 33 | #' description = "my_descr"), 34 | #' fun_path = dir, # as an example 35 | #' fun_name = "my_fun_name", 36 | #' fun_args = list(x = 1, 37 | #' y = 0:4, 38 | #' z = iris), 39 | #' priority = 1) 40 | #' 41 | #' # catch results 42 | #' list.files(conf$dir) 43 | #' read_conf <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 44 | #' y <- readRDS(paste0(conf$dir, "inputs/y.RDS")) 45 | #' z <- readRDS(paste0(conf$dir, "inputs/z.RDS")) 46 | #' 47 | #' } 48 | #' 49 | configure_task <- function(dir_path, 50 | fun_path, 51 | fun_name, 52 | conf_descr = NULL, 53 | fun_args = NULL, 54 | priority = 0L, 55 | compress = TRUE, 56 | call. = TRUE) { 57 | # checks 58 | if (! is.character(dir_path)) { 59 | stop("'dir_path' must be of class .", call. = call.) 60 | } 61 | if (length(dir_path) != 1) { 62 | stop("Only one 'dir_path' accepts", call. = call.) 63 | } 64 | if (! dir.exists(dir_path)) { 65 | stop("'dir_path' directory doesn't exist (", dir_path, ").", call. = call.) 66 | } 67 | 68 | if (! (is.null(conf_descr) || 69 | (is.list(conf_descr) && length(conf_descr) > 0 && 70 | ! is.null(names(conf_descr)) && ! any(names(conf_descr) == "")))) { 71 | stop("'conf_descr' must be a .", call. = call.) 72 | } 73 | if (! is.character(fun_path)) { 74 | stop("'fun_path' must be of class .", call. = call.) 75 | } 76 | if (length(fun_path) != 1) { 77 | stop("Only one 'fun_path' accepts", call. = call.) 78 | } 79 | if (! file.exists(fun_path)) { 80 | stop("'fun_path' file doesn't existed : ", fun_path, call. = call.) 81 | } 82 | 83 | if (! is.character(fun_name)) { 84 | stop("'fun_name' must be of class .", call. = call.) 85 | } 86 | if (length(fun_name) != 1) { 87 | stop("Only one 'fun_name' accepts", call. = call.) 88 | } 89 | if (! (is.null(fun_args) || 90 | (is.list(fun_args) && length(fun_args) > 0 && 91 | ! is.null(names(fun_args)) && ! any(names(fun_args) == "")))) { 92 | stop("'fun_args' must be a of 'fun_name' arguments.", call. = call.) 93 | } 94 | if (! class(priority) %in% c("numeric", "integer")) { 95 | stop("'priority' must be of class or .", call. = call.) 96 | } 97 | 98 | # write conf 99 | sep_path <- "/" 100 | fun_path <- gsub("\\", sep_path, fun_path, fixed = TRUE) 101 | 102 | time <- Sys.time() 103 | dir_path <- gsub("/$", "", gsub("\\", sep_path, dir_path, fixed = TRUE)) 104 | dir_path <- paste0(dir_path, sep_path, 105 | gsub(".", "", format(time, format = "%Y%m%d_%H%M_%OS2"), fixed = TRUE), 106 | sep_path) 107 | 108 | suppressWarnings(dir.create(dir_path, recursive = TRUE)) 109 | if (! dir.exists(dir_path)) { 110 | stop("Can't create output directory ", dir_path, call. = call.) 111 | } 112 | 113 | # paste conf_descr elements 114 | conf_descr <- lapply(conf_descr, function(x){ 115 | paste(x, collapse = ", ") 116 | }) 117 | 118 | conf <- list( 119 | "run_info" = list( 120 | "date_creation" = as.character(time), 121 | "date_start" = "N/A", 122 | "date_end" = "N/A", 123 | "priority" = priority, 124 | "status" = "waiting"), 125 | "descriptive" = conf_descr, 126 | "function" = list( 127 | "path" = fun_path, 128 | "name" = fun_name 129 | ), 130 | "args" = list( 131 | ) 132 | ) 133 | 134 | # retrieve args 135 | if (! is.null(fun_args)) { 136 | for (n_arg in 1:length(fun_args)) { 137 | arg <- fun_args[[n_arg]] 138 | arg_name <- names(fun_args)[[n_arg]] 139 | 140 | # if simple arg, add it to the yaml 141 | if (is.vector(arg) && ! is.list(arg) && length(arg) == 1) { 142 | conf$args[[arg_name]] <- arg 143 | # else save in RDS and add path to the yaml 144 | } else { 145 | input_dir <- paste0(dir_path, "inputs") 146 | if (! dir.exists(input_dir)) { 147 | check_dir <- dir.create(input_dir) 148 | if(!check_dir){ 149 | stop("Can't create output directory ", input_dir) 150 | } 151 | } 152 | 153 | path <- paste0(dir_path, "inputs", sep_path, arg_name, ".RDS") 154 | 155 | saveRDS(arg, file = path, compress = compress) 156 | 157 | conf$args[[arg_name]] <- list("_path" = path) 158 | } 159 | } 160 | } 161 | 162 | # add path to res 163 | conf$dir <- dir_path 164 | 165 | # save yaml 166 | yaml::write_yaml(conf,file = paste0(dir_path, "conf.yml")) 167 | 168 | return(conf) 169 | } 170 | -------------------------------------------------------------------------------- /R/launcher.R: -------------------------------------------------------------------------------- 1 | #' Evaluate and trigger the number of tasks to launch to reach the maximum allowed 2 | #' 3 | #' @param dir_path \code{character}. Where to find the tasks directory (one or more). If several, log file are present in first directory 4 | #' @param max_runs \code{integer}. Maximum number of simultaneous running tasks. 5 | #' @param ignore_status \code{character} (c("running", "finished", "timeout", "error")). Status to be ignored when launching tasks. 6 | #' @param delay_reruns \code{boolean} (TRUE). When "running", "finished", "timeout" or "error" are not in ignore_status, use the date of the last run instead of 7 | #' the date of creation of the task to compute the order of (re)run for these tasks. The priority still applies. 8 | #' @param compress \code{logical or character} (TRUE). Either a logical specifying whether or not to use "gzip" compression, or one of "gzip", "bzip2" or "xz" to indicate the type of compression to be used. 9 | #' @param verbose \code{logical} See running task message ? Default to FALSE 10 | #' @param timeout \code{numeric} Minute. Long task running more than \code{timeout} (perhaps killed with server restart or full memory) are set to "timeout" to enable running other waiting tasks 11 | #' 12 | #' @return the number of launched tasks. 13 | #' 14 | #' @export 15 | #' 16 | #' @import yaml futile.logger 17 | #' 18 | #' @examples 19 | #' 20 | #' \donttest{ 21 | #' 22 | #' # create temporary directory for conf 23 | #' dir_conf <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 24 | #' dir.create(dir_conf, recursive = TRUE) 25 | #' 26 | #' # ex fun 27 | #' fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 28 | #' fun_name = "sb_fun_ex" 29 | #' 30 | #' # create 2 confs 31 | #' conf_1 <- configure_task(dir_path = dir_conf, 32 | #' conf_descr = list( 33 | #' title_1 = "my_title_1", 34 | #' description_1 = "my_descr_1" 35 | #' ), 36 | #' fun_path = fun_path, 37 | #' fun_name = fun_name, 38 | #' fun_args = list( 39 | #' x = 0, 40 | #' y = 0:4, 41 | #' z = iris 42 | #' ), 43 | #' priority = 1) 44 | #' 45 | #' conf_2 <- configure_task(dir_path = dir_conf, 46 | #' conf_descr = list( 47 | #' title_2 = "my_title_2", 48 | #' description_2 = "my_descr_2" 49 | #' ), 50 | #' fun_path = fun_path, 51 | #' fun_name = fun_name, 52 | #' fun_args = list( 53 | #' x = 1, 54 | #' y = 0:4, 55 | #' z = iris 56 | #' ), 57 | #' priority = 2) 58 | #' 59 | #' launcher(dir_conf, verbose = TRUE) 60 | #' # display res of conf_2 in /output dir 61 | #' Sys.sleep(2) # waiting scheduler computation 62 | #' readRDS(paste0(conf_2$dir, "output/res.RDS")) 63 | #' 64 | #' launcher(dir_conf, verbose = TRUE) 65 | #' # display res of conf_1 in /output dir 66 | #' Sys.sleep(2) # waiting scheduler computation 67 | #' readRDS(paste0(conf_1$dir, "output/res.RDS")) 68 | #' 69 | #' launcher(dir_conf, verbose = TRUE) 70 | #' 71 | #' # launch again a finished task 72 | #' launcher(dir_conf, ignore_status = c("running", "error"), verbose = TRUE) 73 | #' 74 | #' log <- read.delim(paste0(dir_conf, "/log_launcher.txt"), header = FALSE) 75 | #' 76 | #' } 77 | #' 78 | launcher <- function(dir_path, 79 | max_runs = 1, 80 | ignore_status = c("running", "finished", "timeout", "error"), 81 | delay_reruns = TRUE, 82 | compress = TRUE, 83 | verbose = FALSE, 84 | timeout = Inf) { 85 | 86 | # checks 87 | if (any(! is.character(dir_path))) { 88 | stop("'dir_path' must be of class .") 89 | } 90 | if (any(! dir.exists(dir_path))) { 91 | stop("'dir_path' directory doesn't exist. (", dir_path, ")") 92 | } 93 | 94 | if(!is.numeric(timeout) | is.integer(timeout)) timeout <- Inf 95 | 96 | # init log 97 | futile.logger::flog.appender(futile.logger::appender.file(paste0(dir_path[1], "/log_launcher.txt")), 98 | name = "launcher.io") 99 | # set layout 100 | layout <- futile.logger::layout.format('[~t] [~l] ~m') 101 | futile.logger::flog.layout(layout, name="launcher.io") 102 | 103 | # and threshold 104 | futile.logger::flog.threshold("INFO", name = "launcher.io") 105 | 106 | nb_to_run <- withCallingHandlers({ 107 | if (verbose) { 108 | message("Starting launcher execution..") 109 | } else { 110 | futile.logger::flog.info("Starting launcher execution...", name = "launcher.io") 111 | } 112 | 113 | if (! (is.numeric(max_runs) && max_runs > 0)) { 114 | stop("'max_runs' must be a positive integer.", call. = FALSE) 115 | } else { 116 | max_runs <- round(max_runs) 117 | } 118 | if (is.null(ignore_status)) ignore_status <- "" 119 | if (! is.character(ignore_status)) { 120 | stop("'ignore_status' must be of class .", call. = FALSE) 121 | } 122 | 123 | # retrieve conf files 124 | confs <- tryCatch({ 125 | do.call("c", lapply(dir_path, function(p) { 126 | conf_paths <- if (! is.null(p)) { 127 | list.dirs(p, full.names = TRUE, recursive = FALSE) 128 | } else { 129 | NULL 130 | } 131 | 132 | lapply(conf_paths, function(x) { 133 | yaml::read_yaml(paste0(x, "/conf.yml")) 134 | }) 135 | })) 136 | }, 137 | error = function(e) { 138 | stop(paste0("Error reading configuration files : ", e$message), call. = FALSE) 139 | }) 140 | 141 | # run tasks 142 | nb_to_run <- 0 143 | 144 | if (verbose) { 145 | message(paste0("Number of detected conf files : ", length(confs), ".")) 146 | } else { 147 | futile.logger::flog.info(message(paste0("Number of detected conf files : ", length(confs), "."))) 148 | } 149 | 150 | if (length(confs) > 0) { 151 | tbl_global <- conf_to_dt(confs = confs, 152 | allowed_run_info_cols = c("date_creation", "date_start", "date_end", "priority", "status"), 153 | allow_descr = FALSE, 154 | allowed_function_cols = "", 155 | allow_args = FALSE)$tbl_global 156 | 157 | nb_running <- sum(tbl_global$status == "running") 158 | nb_to_run <- min(max_runs - nb_running, sum(! tbl_global$status %in% ignore_status)) 159 | 160 | # timeout 161 | n_timeout <- 0 162 | if(nb_running > 0 & !is.infinite(timeout) & timeout > 0){ 163 | # saveRDS(tbl_global, "/home/bthieurmel/tbl_global.RDS") 164 | 165 | is_running <- which(tbl_global$status == "running") 166 | for(i in is_running){ 167 | run_time <- as.numeric(difftime(Sys.time(), tbl_global$starting_date[i], units = "mins")) 168 | message(run_time) 169 | if(run_time > timeout){ 170 | conf_path <- file.path(tbl_global$dir[i], "conf.yml") 171 | conf <- tryCatch(yaml::read_yaml(conf_path), 172 | error = function(e) { 173 | stop(paste0("Error reading '", conf_path, "' : ", e$message), call. = FALSE) 174 | }) 175 | 176 | conf$run_info$status <- "timeout" 177 | 178 | yaml::write_yaml(conf,file = conf_path) 179 | 180 | n_timeout <- n_timeout + 1 181 | nb_running <- nb_running - 1 182 | } 183 | } 184 | nb_to_run <- min(max_runs - nb_running, sum(! tbl_global$status %in% ignore_status)) 185 | } 186 | 187 | if (verbose) { 188 | message(paste0("Number of tasks available for a run : ", sum(! tbl_global$status %in% ignore_status), ".")) 189 | message(paste0("Maximum number of simultaneous runs : ", max_runs, ".")) 190 | message(paste0("Number of currently running tasks : ", sum(tbl_global$status == "running"), ".")) 191 | message(paste0("Number of tasks to be started : ", max(0, nb_to_run), ".")) 192 | message(paste0("Number of tasks passed as timeout : ", n_timeout, ".")) 193 | } else { 194 | futile.logger::flog.info(message(paste0("Number of tasks available for a run : ", sum(! tbl_global$status %in% ignore_status), "."))) 195 | futile.logger::flog.info(message(paste0("Maximum number of simultaneous runs : ", max_runs, "."))) 196 | futile.logger::flog.info(message(paste0("Number of currently running tasks : ", sum(tbl_global$status == "running"), "."))) 197 | futile.logger::flog.info(message(paste0("Number of tasks to be started : ", max(0, nb_to_run), "."))) 198 | futile.logger::flog.info(message(paste0("Number of tasks passed as timeout : ", n_timeout, "."))) 199 | } 200 | 201 | if (nb_to_run > 0) { 202 | run_order_ <- run_order(confs = confs, 203 | ignore_status = ignore_status, 204 | delay_reruns = delay_reruns) 205 | 206 | for (i in 1:nb_to_run) { 207 | os <- Sys.info()[['sysname']] 208 | 209 | # retrieve OS rscript_path 210 | if (os == "Windows") { 211 | rscript_path <- file.path(Sys.getenv("R_HOME"), "bin", "Rscript.exe") 212 | } else { 213 | rscript_path <- file.path(Sys.getenv("R_HOME"), "bin", "Rscript") 214 | } 215 | 216 | if (! file.exists(rscript_path)) { 217 | stop("Could not find Rscript, thus could not launch the batch task.") 218 | 219 | } else { 220 | # create cmd 221 | fun_call <- paste0("run_task(conf_path = '", paste0(confs[[run_order_[i]]]$dir, "conf.yml"), "'", 222 | ", ignore_status = c('", paste0(ignore_status, collapse = "', '"), "')", 223 | ", verbose = ", verbose, 224 | ", compress = ", compress, 225 | ", return = FALSE)") 226 | 227 | cmd <- paste0(rscript_path, 228 | " --vanilla -e \"{", 229 | "require(shinybatch) ; ", 230 | fun_call, " ;}\"") 231 | 232 | # run in batch 233 | system(cmd, intern = FALSE, wait = FALSE, ignore.stdout = TRUE, ignore.stderr = TRUE) 234 | 235 | if (verbose) { 236 | message(paste0("Task launched (", i, "/", nb_to_run, ") : ", 237 | rev(strsplit(confs[[run_order_[i]]]$dir, split = "/")[[1]])[1]), ".") 238 | message(paste0("Task priority : ", confs[[run_order_[i]]]$run_info$priority, 239 | " ; task status : ", confs[[run_order_[i]]]$run_info$status, ".")) 240 | } else { 241 | futile.logger::flog.info(message(paste0("Task launched (", i, "/", nb_to_run, ") : ", 242 | rev(strsplit(confs[[run_order_[i]]]$dir, split = "/")[[1]])[1]), ".")) 243 | futile.logger::flog.info(message(paste0("Task priority : ", confs[[run_order_[i]]]$run_info$priority, 244 | " ; task status : ", confs[[run_order_[i]]]$run_info$status, "."))) 245 | } 246 | } 247 | } 248 | } 249 | } 250 | 251 | if (verbose) { 252 | message("... launcher terminated.") 253 | } else { 254 | futile.logger::flog.info("... launcher terminated.", name = "launcher.io") 255 | } 256 | 257 | nb_to_run 258 | 259 | }, simpleError = function(e) { 260 | futile.logger::flog.fatal(gsub("^(Error in withCallingHandlers[[:punct:]]{3}[[:space:]]*)|(\n)*$", "", e), name="launcher.io") 261 | 0 262 | 263 | }, warning = function(w) { 264 | futile.logger::flog.warn(gsub("(\n)*$", "", w$message), name = "launcher.io") 265 | }, message = function(m) { 266 | futile.logger::flog.info(gsub("(\n)*$", "", m$message), name = "launcher.io") 267 | }) 268 | 269 | return(nb_to_run) 270 | } 271 | -------------------------------------------------------------------------------- /R/module_configure_task.R: -------------------------------------------------------------------------------- 1 | #' Module shiny to configure a task. 2 | #' 3 | #' @param input shiny input 4 | #' @param output shiny input 5 | #' @param session shiny input 6 | #' @param btn \code{reactive}. Link to a actionButton for call \code{configure_task} function 7 | #' @param dir_path \code{character}. Tasks location (parent directory). 8 | #' @param conf_descr \code{named list} (NULL). Optional description fields. 9 | #' @param fun_path \code{character}. Path to the script of the function. 10 | #' @param fun_name \code{character}. Name of the function in fun_path script. 11 | #' @param fun_args \code{named list} (NULL). Args of the function, must all be named. 12 | #' @param priority \code{numeric} (0L). Number used to define which task should be launched first. 13 | #' @param compress \code{logical or character} (TRUE). Either a logical specifying whether or not to use "gzip" compression, or one of "gzip", "bzip2" or "xz" to indicate the type of compression to be used. 14 | #' @param labels \code{list}. Modal dialog title. 15 | #' 16 | #' @return Nothing. 17 | #' 18 | #' @export 19 | #' 20 | #' @import shiny 21 | #' 22 | #' @seealso \code{\link[shinybatch]{tasks_overview_server}} 23 | #' 24 | #' @examples 25 | #' 26 | #' \donttest{ 27 | #' 28 | #' if(interactive()){ 29 | #' 30 | #' require(shiny) 31 | #' 32 | #' # create temporary directory for conf 33 | #' dir_conf <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 34 | #' dir.create(dir_conf, recursive = TRUE) 35 | #' 36 | #'# ex fun 37 | #' fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 38 | #' fun_name = "sb_fun_ex" 39 | #' 40 | #' # create and save conf 41 | #' ui <- shiny::fluidPage( 42 | #' fluidRow( 43 | #' column(offset = 3, 6, 44 | #' shiny::actionButton("conf_task", "Configure the task", width = "100%") 45 | #' ) 46 | #' ) 47 | #' ) 48 | #' server <- function(input, output, session) { 49 | #' callModule(configure_task_server, "my_id_1", 50 | #' btn = reactive(input$conf_task), 51 | #' dir_path = dir_conf, 52 | #' conf_descr = list(title = "my_title", 53 | #' description = "my_descr"), 54 | #' fun_path = fun_path, 55 | #' fun_name = fun_name, 56 | #' fun_args = list(x = 1, 57 | #' y = 0:4, 58 | #' z = iris), 59 | #' priority = 1) 60 | #' } 61 | #' shiny::shinyApp(ui = ui, server = server) 62 | #' 63 | #' # catch results 64 | #' list.files(path <- list.dirs(dir_conf, full.names = TRUE, recursive = FALSE)) 65 | #' path 66 | #' read_conf <- yaml::read_yaml(paste0(path[1], "/", "conf.yml")) 67 | #' y <- readRDS(paste0(path[1], "/", "inputs/y.RDS"));y 68 | #' z <- readRDS(paste0(path[1], "/", "inputs/z.RDS"));z 69 | #' 70 | #' } 71 | #' } 72 | #' 73 | #' @rdname module_configure_task 74 | #' 75 | configure_task_server <- function(input, output, session, 76 | btn, 77 | dir_path, 78 | fun_path, 79 | fun_name, 80 | conf_descr = NULL, 81 | fun_args = NULL, 82 | priority = 0L, 83 | compress = TRUE, 84 | labels = list( 85 | success = "Task configured !", 86 | error = "Error when configuring the task" 87 | )) { 88 | 89 | # reactive controls 90 | if (! shiny::is.reactive(dir_path)) { 91 | get_dir_path <- shiny::reactive(dir_path) 92 | } else { 93 | get_dir_path <- dir_path 94 | } 95 | if (! shiny::is.reactive(conf_descr)) { 96 | get_conf_descr <- shiny::reactive(conf_descr) 97 | } else { 98 | get_conf_descr <- conf_descr 99 | } 100 | if (! shiny::is.reactive(fun_path)) { 101 | get_fun_path <- shiny::reactive(fun_path) 102 | } else { 103 | get_fun_path <- fun_path 104 | } 105 | if (! shiny::is.reactive(fun_name)) { 106 | get_fun_name <- shiny::reactive(fun_name) 107 | } else { 108 | get_fun_name <- fun_name 109 | } 110 | if (! shiny::is.reactive(fun_args)) { 111 | get_fun_args <- shiny::reactive(fun_args) 112 | } else { 113 | get_fun_args <- fun_args 114 | } 115 | if (! shiny::is.reactive(priority)) { 116 | get_priority <- shiny::reactive(priority) 117 | } else { 118 | get_priority <- priority 119 | } 120 | if (! shiny::is.reactive(compress)) { 121 | get_compress <- shiny::reactive(compress) 122 | } else { 123 | get_compress <- compress 124 | } 125 | if (! shiny::is.reactive(labels)) { 126 | get_labels <- shiny::reactive(labels) 127 | } else { 128 | get_labels <- labels 129 | } 130 | # check args 131 | output$is_args <- reactive({ 132 | ! (is.null(get_dir_path()) || 133 | is.null(get_fun_path()) || 134 | is.null(get_fun_name())) 135 | }) 136 | outputOptions(output, "is_args", suspendWhenHidden = FALSE) 137 | 138 | observe({ 139 | btn <- btn() 140 | 141 | isolate({ 142 | if (btn > 0) { 143 | try <- try(configure_task(dir_path = get_dir_path(), 144 | conf_descr = get_conf_descr(), 145 | fun_path = get_fun_path(), 146 | fun_name = get_fun_name(), 147 | fun_args = get_fun_args(), 148 | priority = get_priority(), 149 | compress = get_compress(), call. = FALSE), silent = TRUE) 150 | 151 | if ("try-error" %in% class(try)) { 152 | showModal( 153 | modalDialog( 154 | easyClose = TRUE, 155 | footer = NULL, 156 | title = tags$p(ifelse(is.null(get_labels()$error), 157 | "Error when configuring the task", 158 | get_labels()$error), style = "color:red;"), 159 | try[[1]] 160 | ) 161 | ) 162 | } else { 163 | showModal( 164 | modalDialog( 165 | size = "s", 166 | easyClose = TRUE, 167 | footer = NULL, 168 | tags$h4(ifelse(is.null(get_labels()$success), 169 | "Task configured !", 170 | get_labels()$success), style = "color:green;") 171 | ) 172 | ) 173 | } 174 | } 175 | }) 176 | }) 177 | } 178 | -------------------------------------------------------------------------------- /R/run_order.R: -------------------------------------------------------------------------------- 1 | #' Compute the order of priority to run a list of configurations 2 | #' 3 | #' @param confs \code{list}. List of conf tables. 4 | #' @param ignore_status \code{character} (c("running", "finished", "timeout", "error")). Status to be ignored when launching tasks. 5 | #' @param delay_reruns \code{boolean} (TRUE). When "running", "finished", "timeout" or "error" are not in ignore_status, use the date of the last run instead of 6 | #' the date of creation of the task to compute the order of (re)run for these tasks. The priority still applies. 7 | #' 8 | #' @import data.table 9 | #' 10 | #' @details The order is determined first by the priority and then by the date of creation of the task. 11 | #' 12 | #' @return the order of priority of the given confs according to their priority argument and 13 | #' date of creation. 14 | #' 15 | #' 16 | #' @export 17 | run_order <- function(confs, 18 | ignore_status = c("running", "finished", "timeout", "error"), 19 | delay_reruns = TRUE) { 20 | 21 | if (delay_reruns) { 22 | conf_delayed <- sapply(confs, function(x) x$run_info$status != "waiting") 23 | confs[conf_delayed] <- lapply(confs[conf_delayed], function(x) {x$run_info$date_creation <- x$run_info$date_start ; x}) 24 | } 25 | 26 | valid_status <- sapply(confs, function(x) (! x$run_info$status %in% ignore_status)) 27 | priority <- sapply(confs, function(x) x$run_info$priority) 28 | date_creation <- sapply(confs, function(x) x$run_info$date_creation) 29 | 30 | data_order <- data.table::data.table("order" = 1:length(confs), 31 | "valid_status" = valid_status, 32 | "date_creation" = date_creation, 33 | "priority" = priority) 34 | 35 | run_order <- data_order[order(date_creation) 36 | ][order(priority, decreasing = TRUE) 37 | ][order(valid_status, decreasing = TRUE) 38 | ][["order"]] 39 | 40 | return(run_order) 41 | } 42 | -------------------------------------------------------------------------------- /R/run_task.R: -------------------------------------------------------------------------------- 1 | #' Run the task defined in a conf file 2 | #' 3 | #' @param conf_path \code{character}. Path to the conf file. 4 | #' @param ignore_status \code{character} (c("running", "finished", "timeout", "error")). Status to be ignored when launching tasks. 5 | #' @param save_rds \code{logical} Save output in output/res.RDS ? Default to TRUE 6 | #' @param compress \code{logical or character} (TRUE). Either a logical specifying whether or not to use "gzip" compression, or one of "gzip", "bzip2" or "xz" to indicate the type of compression to be used. 7 | #' @param return \code{logical} Get back result in R ? Default to TRUE 8 | #' @param verbose \code{logical} See running task message ? Default to FALSE 9 | #' 10 | #' @return the result of the task (function applied on prepared args). 11 | #' @export 12 | #' 13 | #' @import yaml futile.logger 14 | #' 15 | #' @examples 16 | #' 17 | #' \donttest{ 18 | #' 19 | #' # create temporary directory for conf 20 | #' dir_conf <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 21 | #' dir.create(dir_conf, recursive = TRUE) 22 | #' 23 | #' # ex fun 24 | #' fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 25 | #' fun_name = "sb_fun_ex" 26 | #' 27 | #' # create and save conf 28 | #' conf <- configure_task(dir_path = dir_conf, 29 | #' conf_descr = list( 30 | #' title = "my_title", 31 | #' description = "my_descr" 32 | #' ), 33 | #' fun_path = fun_path, 34 | #' fun_name = fun_name, 35 | #' fun_args = list( 36 | #' x = 1, 37 | #' y = 0:4, 38 | #' z = iris 39 | #' ), 40 | #' priority = 1) 41 | #' 42 | #' conf_init <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 43 | #' y <- readRDS(paste0(conf$dir, "inputs/y.RDS")) 44 | #' z <- readRDS(paste0(conf$dir, "inputs/z.RDS")) 45 | #' 46 | #' run_task(paste0(conf$dir, "conf.yml")) 47 | #' 48 | #' # catch results 49 | #' list.files(conf$dir) 50 | #' conf_update <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 51 | #' output <- readRDS(paste0(conf$dir, "output/res.RDS")) 52 | #' log <- read.delim(list.files(paste0(conf$dir, "output/"), 53 | #' pattern = "log_run", full.names = TRUE), header = FALSE) 54 | #' 55 | #' } 56 | run_task <- function(conf_path, 57 | ignore_status = c("running", "finished", "timeout", "error"), 58 | save_rds = TRUE, compress = TRUE, return = TRUE, verbose = FALSE) { 59 | 60 | current_wd <- getwd() 61 | on.exit(setwd(current_wd)) 62 | 63 | # checks 64 | if (! is.character(conf_path)) { 65 | stop("'conf_path' must be of class .", call. = FALSE) 66 | } 67 | if (! file.exists(conf_path)) { 68 | stop("'conf_path' file doesn't exist. (", conf_path, ")", call. = FALSE) 69 | } 70 | if(is.null(ignore_status)) ignore_status <- "" 71 | if (! is.character(ignore_status)) { 72 | stop("'ignore_status' must be of class .", call. = FALSE) 73 | } 74 | 75 | stopifnot(is.logical(save_rds)) 76 | stopifnot(is.logical(return)) 77 | 78 | # read conf 79 | conf <- tryCatch(yaml::read_yaml(conf_path), 80 | error = function(e) { 81 | stop(paste0("Error reading '", conf_path, "' : ", e$message), call. = FALSE) 82 | }) 83 | 84 | fun_res <- NULL 85 | 86 | if (! conf$run_info$status %in% ignore_status) { 87 | dir_result <- paste0(dirname(conf_path), "/output") 88 | if(!dir.exists(dir_result)){ 89 | check_dir <- dir.create(dir_result) 90 | if (! check_dir) { 91 | stop("Cannot create output directory ", paste0(dirname(conf_path), "/output"), call. = FALSE) 92 | } 93 | } 94 | 95 | # init log 96 | time <- Sys.time() 97 | format_time <- gsub(".", "", format(time, format = "%Y%m%d_%H%M_%OS2"), fixed = TRUE) 98 | futile.logger::flog.appender(futile.logger::appender.file(paste0(dirname(conf_path), "/output/log_run_", format_time, ".txt")), 99 | name = "run_task.io") 100 | # set layout 101 | layout <- futile.logger::layout.format('[~t] [~l] ~m') 102 | futile.logger::flog.layout(layout, name="run_task.io") 103 | 104 | # and threshold 105 | futile.logger::flog.threshold("INFO", name = "run_task.io") 106 | 107 | fun_res <- withCallingHandlers({ 108 | 109 | if(verbose){ 110 | message("Starting task execution...") 111 | } else { 112 | futile.logger::flog.info("Starting task execution...", name = "run_task.io") 113 | } 114 | 115 | conf$run_info$date_start <- as.character(time) 116 | conf$run_info$status <- "running" 117 | yaml::write_yaml(conf,file = conf_path) 118 | 119 | # retrieve fun args 120 | fun_args <- conf$args 121 | if (length(fun_args) > 0) { 122 | for (n_arg in 1:length(fun_args)) { 123 | arg <- fun_args[[n_arg]] 124 | 125 | if (! is.null(names(arg)) && names(arg) == "_path") { 126 | # list() for NULL Case 127 | fun_args[n_arg] <- list( 128 | tryCatch(readRDS(arg[["_path"]]), 129 | error = function(e) { 130 | stop(paste0("Path '", arg[["_path"]], "' doesnt exist.")) 131 | }) 132 | ) 133 | } 134 | } 135 | } 136 | 137 | # apply fun + create log 138 | tryCatch(source(conf[["function"]]$path), 139 | error = function(e) { 140 | stop(paste0("File '", conf[["function"]]$path, "' cannot be sourced."), call. = FALSE) 141 | }) 142 | 143 | setwd(paste0(dirname(conf_path), "/output")) 144 | 145 | # run fun 146 | 147 | # add a try to catch exception (for instance when failing to run code in Python with reticulate) 148 | res <- try(do.call(conf[["function"]]$name, fun_args), silent = TRUE) 149 | if ("try-error" %in% class(res)) { 150 | stop(attr(res, "condition")$message) 151 | } 152 | 153 | if(verbose){ 154 | message("... task terminated.") 155 | } else { 156 | futile.logger::flog.info("... task terminated.", name = "run_task.io") 157 | } 158 | res 159 | 160 | }, simpleError = function(e){ 161 | futile.logger::flog.fatal(gsub("^(Error in withCallingHandlers[[:punct:]]{3}[[:space:]]*)|(\n)*$", "", e$message), name="run_task.io") 162 | 163 | # update conf file if error 164 | conf$run_info$date_end <- as.character(Sys.time()) 165 | conf$run_info$status <- "error" 166 | 167 | yaml::write_yaml(conf,file = conf_path) 168 | 169 | }, warning = function(w){ 170 | futile.logger::flog.warn(gsub("(\n)*$", "", w$message), name = "run_task.io") 171 | }, message = function(m){ 172 | futile.logger::flog.info(gsub("(\n)*$", "", m$message), name = "run_task.io") 173 | }) 174 | 175 | if(save_rds){ 176 | saveRDS(fun_res, 177 | file = paste0(dirname(conf_path), "/output/res.RDS"), 178 | compress = compress) 179 | } 180 | 181 | # update conf file 182 | conf$run_info$date_end <- as.character(Sys.time()) 183 | conf$run_info$status <- "finished" 184 | 185 | yaml::write_yaml(conf, file = conf_path) 186 | } 187 | 188 | if(return){ 189 | return(fun_res) 190 | } else { 191 | return(invisible(NULL)) 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /R/utils.R: -------------------------------------------------------------------------------- 1 | #' create button for DT table 2 | #' 3 | #' @param inputId \code{character}.\code{\link{actionButton}}. 4 | #' @param col_value \code{vector}. \code{\link{actionButton}}. 5 | #' @param tooltip \code{character}. \code{\link{actionButton}}. 6 | #' @param label \code{character}. \code{\link{actionButton}}. 7 | #' @param icon \code{character}. \code{\link{actionButton}}. 8 | #' @param status \code{character}. \code{\link{actionButton}}. 9 | #' 10 | #' @return a shiny btn. 11 | #' 12 | #' @import shiny htmltools 13 | #' 14 | input_btns <- function(inputId, 15 | col_value, 16 | tooltip, 17 | label = "", 18 | icon = NULL, 19 | status = "primary") { 20 | 21 | tag <- lapply( 22 | X = col_value, 23 | FUN = function(x) { 24 | res <- tags$button( 25 | class = paste0("btn btn-", status), 26 | style = "float: right;", 27 | onclick = sprintf( 28 | "Shiny.setInputValue('%s', '%s', {priority: 'event'})", 29 | inputId, x 30 | ), 31 | label, 32 | icon, 33 | `data-toggle` = "tooltip", 34 | `data-title` = tooltip, 35 | `data-container` = "body" 36 | ) 37 | 38 | res <- tagList(res, tags$script(HTML("$('[data-toggle=\"tooltip\"]').tooltip();"))) 39 | htmltools::doRenderTags(res) 40 | } 41 | ) 42 | 43 | unlist(tag, use.names = FALSE) 44 | } 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shinybatch 2 | 3 | This package provides a simple framework to create, launch *automatically* and retrieve *time-consuming operations* (tasks) **in batch mode** from a **Shiny app**. 4 | 5 | The tasks are automatically launched using a scheduler, e.g. a timer that periodically launches a (batch) operation. 6 | 7 | - with a CRON in Linux/Mac, using package **cronR** 8 | - with Windows Task Scheduler in Windows, using package **taskscheduleR** 9 | - or defined directly with OS tools 10 | 11 | ### Installation 12 | 13 | ``` r 14 | devtools::install_github("datastorm-open/shinybatch") 15 | ``` 16 | ### Demo application 17 | 18 | ``` r 19 | runApp(system.file("demo_app", package = "shinybatch")) 20 | ``` 21 | 22 | ### Main functions 23 | 24 | - **configure_task** : create a *.yml* file filed with operation params (fun path, fun args, priority, ...), 25 | - **run_task** : run a selected task from a *.yml* file. Used by **launcher**, 26 | - **launcher** : select and run the task(s) with highest priority, calling **run_task** in batch mode with *RScript* command, 27 | - **scheduler_init** : (opt: create the cron R script) create the file to be launched by the *scheduler*, 28 | - **scheduler_add** : (opt: create the cron file, and by default the cron R script) and start the *scheduler* which will read the file at the given frequency to launch batch operations, 29 | - **configure_task_server** : define a task and call *configure_task()* in the Shiny app, 30 | - **tasks_overview_server** : display tasks & retrieve results within the Shiny app. 31 | 32 | 33 | ### Definition of a task 34 | 35 | *configure_task()* 36 | 37 | A task is defined by its *.yml* file that contains the following informations : 38 | 39 | ``` yml 40 | run_info: 41 | date_creation: 2020-04-24 15:21:00 42 | date_start: N/A 43 | date_end: N/A 44 | priority: 1.0 45 | status: waiting 46 | descriptive: 47 | title: my_title 48 | description: my_descr 49 | function: 50 | path: /path/to/my_fun 51 | name: my_fun_name 52 | args: 53 | x: 1.0 54 | 'y': 55 | _path: /path/to/task/dir/inputs/y.RDS 56 | z: 57 | _path: /path/to/task/dir/inputs/z.RDS 58 | dir: /path/to/task/dir/ 59 | ``` 60 | 61 | 62 | The ``run_info`` part contains general informations about the task. 63 | 64 | Priority can be any number, with 0 as default. The highest the priority, the sooner it is launched. 65 | 66 | Valid status are: 67 | 68 | - **waiting** 69 | - **running** 70 | - **finished** 71 | - **timeout** 72 | - **error** 73 | 74 | The ``descriptive`` part contains free informations given by the user. The title and description fields are only example. 75 | 76 | The ``function`` part contains the location of the fun **R** script (for sourcing) and its name (for calling). The script must have all necessary ressources (packages, variables, functions) for execute the main function : 77 | 78 | ``` r 79 | # Load package(s) (if needed) 80 | require(data.table) 81 | 82 | # source script(s) (if needed) 83 | source("/path/to/script") 84 | 85 | # Load data (if needed) 86 | data <- readRDS("/path/to/script") 87 | 88 | # Define main function (needed !) 89 | my_fun_name <- function(x, y, z){ 90 | ... 91 | } 92 | ``` 93 | 94 | The ``args`` part contains either the argument itself if it is of length 1 or the location of the argument (in *dir_conf/inputs/arg_name.RDS*), complex arguments are storing as *.RDS* 95 | 96 | The ``dir`` argument contains the location of the directory in which is stored the *conf.yml* file. 97 | 98 | When a task has been succesfully run, some fields are updated: 99 | 100 | - ``date_start`` and ``date_end`` are filled, 101 | - ``status`` is set to 'running', then to 'finished'. 102 | 103 | Some outputs are created: 104 | 105 | - if wanted, the result of the task, in other words the output of the main function (in *dir_conf/output/res.RDS*) 106 | - the log of the run (in *dir_conf/output/log_run.txt*) 107 | 108 | ### Description of the launcher 109 | 110 | *launcher()* 111 | 112 | The launcher retrieves all the tasks in a main directory and build a the table of their *run_info*. Based on this, it verifies that there are tasks with *status* that allow a run, e.g. all but those in *ignore_status*. Then, if the maximum number of simultaneously running tasks is not reached, it launches new tasks according to their priority, in batch mode with *RScript* command (and calling *run_task()*) 113 | 114 | 115 | The task with higher priority is defined as the one: 116 | 117 | - with *status* not in *ignore_status* (default is all but **waiting**), 118 | - which has the highest *priority*, 119 | - and then the oldest *date_init*. 120 | 121 | ### Description of the scheduler 122 | 123 | Before calling the scheduler, we first need to create the file that it will launch. We can use **scheduler_init()**. By default, it looks like this (*scheduler_script.R*) : 124 | 125 | ```` 126 | #!/usr/bin/env Rscript 127 | args = commandArgs(trailingOnly = TRUE) 128 | 129 | shinybatch::launcher(dir_path = '/path/to/main_directory/', 130 | max_runs = 1, 131 | ignore_status = c('running','finished','error'), 132 | delay_reruns = TRUE 133 | ) 134 | ```` 135 | 136 | ... but the head lines can be customized by filling the *head_rows* argument. 137 | 138 | 139 | Once the file has been created, the cron has to be defined. You can use directly ``cron`` on linux or the ``Task Scheduler`` on windows, or the **scheduler_add** function. The default command is : 140 | 141 | ``Rscript /path/to/scheduler_script.R`` 142 | 143 | *N.B :* **scheduler_add** creates by default the **R** scheduler script using **scheduler_init** 144 | 145 | ### Example 146 | 147 | **sb_fun_ex.R** 148 | 149 | ``` r 150 | sb_fun_ex <- function(x, y, z) { 151 | res <- x + y 152 | message('Running !') 153 | warning("Complex (or not) variable z is not used...!", call. = FALSE) 154 | res 155 | } 156 | ``` 157 | 158 | **Configure a task** 159 | ``` r 160 | require(shinybatch) 161 | 162 | ?configure_task 163 | 164 | # create temporary directory 165 | dir <- tempdir() 166 | 167 | # create and save conf 168 | conf <- configure_task( 169 | dir_path = dir, 170 | conf_descr = list( 171 | title = "my_title", 172 | description = "my_descr" 173 | ), 174 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch"), # as an example, 175 | fun_name = "sb_fun_ex", 176 | fun_args = list( 177 | x = 1, 178 | y = 0:4, 179 | z = iris 180 | ), 181 | priority = 1) 182 | 183 | # check results 184 | list.files(conf$dir, recursive = T) 185 | # [1] "conf.yml" "inputs/y.RDS" "inputs/z.RDS" 186 | 187 | read_conf <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 188 | 189 | read_conf$run_info 190 | # $date_creation 191 | # [1] "2021-03-02 16:54:24" 192 | # 193 | # $date_start 194 | # [1] "N/A" 195 | # 196 | # $date_end 197 | # [1] "N/A" 198 | # 199 | # $priority 200 | # [1] 1 201 | # 202 | # $status 203 | # [1] "waiting" 204 | 205 | read_conf$args$x 206 | # [1] 1 207 | 208 | y <- readRDS(paste0(conf$dir, "inputs/y.RDS")) 209 | # [1] 0 1 2 3 4 210 | 211 | z <- readRDS(paste0(conf$dir, "inputs/z.RDS")) 212 | ``` 213 | 214 | **Run one given task (for demo/test)** 215 | 216 | ``` r 217 | ?run_task 218 | 219 | run_task(paste0(conf$dir, "conf.yml")) 220 | 221 | # catch results 222 | list.files(conf$dir, recursive = T) 223 | # output directory with log & result 224 | # [1] "conf.yml" "inputs/y.RDS" 225 | # [3] "inputs/z.RDS" "output/log_run_20210302_1656_1080.txt" 226 | # [5] "output/res.RDS" 227 | 228 | conf_update <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 229 | conf_update$run_info 230 | # $date_creation 231 | # [1] "2021-03-02 16:54:24" 232 | # 233 | # $date_start 234 | # [1] "2021-03-02 16:56:10" 235 | # 236 | # $date_end 237 | # [1] "2021-03-02 16:56:10" 238 | # 239 | # $priority 240 | # [1] 1 241 | # 242 | # $status 243 | # [1] "finished" 244 | 245 | 246 | output <- readRDS(paste0(conf$dir, "output/res.RDS")) 247 | #[1] 1 2 3 4 5 248 | 249 | log <- read.delim(list.files(paste0(conf$dir, "output/"), pattern = "log_run", full.names = T), header = F) 250 | # [2021-03-02 16:56:10] [INFO] Starting task execution... 251 | # [2021-03-02 16:56:10] [INFO] Running ! 252 | # [2021-03-02 16:56:10] [WARN] Complex (or not) variable z is not used...! 253 | # [2021-03-02 16:56:10] [INFO] ... task terminated. 254 | ``` 255 | 256 | **Use scheduler to launch the tasks** 257 | 258 | ``` r 259 | ?scheduler_add 260 | 261 | # create temporary directory for conf 262 | dir_conf <- paste0(tempdir(), "/conf/") 263 | dir.create(dir_conf, recursive = T) 264 | 265 | # create 2 confs 266 | conf_1 <- configure_task( 267 | dir_path = dir_conf, 268 | conf_descr = list( 269 | title_1 = "my_title_1", 270 | description_1 = "my_descr_1" 271 | ), 272 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch"), # as an example, 273 | fun_name = "sb_fun_ex", 274 | fun_args = list( 275 | x = 0, 276 | y = 0:4, 277 | z = iris 278 | ), 279 | priority = 1) 280 | 281 | conf_2 <- configure_task( 282 | dir_path = dir_conf, 283 | conf_descr = list( 284 | title_1 = "my_title_2", 285 | description_1 = "my_descr_2" 286 | ), 287 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch"), # as an example, 288 | fun_name = "sb_fun_ex", 289 | fun_args = list( 290 | x = 0, 291 | y = 0:4, 292 | z = iris 293 | ), 294 | priority = 2) 295 | 296 | # on LINUX -> Needs cronR package 297 | # on Windows -> Needs taskscheduleR package 298 | 299 | scheduler_add( 300 | dir_scheduler = tempdir(), 301 | dir_conf = dir_conf, 302 | max_runs = 1, 303 | create_file = T, 304 | head_rows = NULL, 305 | taskname = "cron_script_ex" 306 | ) 307 | 308 | scheduler_ls() # display existing crons 309 | 310 | # wait up to 1 min for conf_2 and up to 2 mins for conf_1 311 | yaml::read_yaml(paste0(conf_1$dir, "/conf.yml"))$run_info$status 312 | yaml::read_yaml(paste0(conf_2$dir, "/conf.yml"))$run_info$status 313 | 314 | scheduler_rm(id = "cron_script_ex") # kill selected cron 315 | ``` 316 | 317 | **Shiny modules** 318 | 319 | These modules contain the basic framework to use all the previous functions in a Shiny app. 320 | Both are used in the demo app which presents a simple usecase. 321 | 322 | - **Configure a new task** 323 | 324 | 325 | 326 | 327 | 328 | call: 329 | 330 | ``` r 331 | ?configure_task_server 332 | 333 | # ui : just create an actionButton 334 | actionButton("go_task", "Configure the task !") 335 | 336 | # server 337 | # call module to configure a task 338 | # connect app inputs to the module 339 | callModule(configure_task_server, "my_id_1", 340 | btn = reactive(input$go_task), 341 | dir_path = dir_conf, 342 | conf_descr = reactive( 343 | list( 344 | "title" = input$title, 345 | "description" = input$description 346 | ) 347 | ), 348 | fun_path = paste0(dir_fun, "/fun_script.R"), 349 | fun_name = "my_fun", 350 | fun_args = reactive( 351 | list( 352 | n = input$fun_nb_points, 353 | mean = input$fun_mean, 354 | sd = input$fun_sd, 355 | sleep = input$sleep_time 356 | ) 357 | ), 358 | priority = reactive(input$priority) 359 | ) 360 | ``` 361 | 362 | 363 | - **Display configured tasks** 364 | 365 | 366 | 367 | 368 | 369 | call: 370 | 371 | ``` r 372 | ?tasks_overview_UI 373 | 374 | # ui 375 | tasks_overview_UI("my_id_2") 376 | 377 | # server 378 | # call module to view tasks 379 | sel_task <- callModule( 380 | tasks_overview_server, "my_id_2", 381 | dir_path = dir_conf, 382 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 383 | allowed_run_info_cols = NULL, 384 | allowed_function_cols = NULL, 385 | allow_descr = T, 386 | allow_args = T 387 | ) 388 | ``` 389 | 390 | This module returns : 391 | 392 | - the status of the selected line (one run) of the summary table, 393 | - the path to the directory in which its output is stored. 394 | 395 | Thus we know when a run is finished and we can load its result to reuse/display it : (readRDS(paste0(path, "/res.RDS"))). 396 | 397 | **Demo app** 398 | 399 | A demo app to create and automatically launch an example task : the generation of normally distributed observations. 400 | 401 | - **global** : the path to the confs directory, the path to the script of the function to be run, the call to scheduler_add() ; 402 | - **ui** : shiny inputs (description args for the conf ; parameters for the function to be called by the cron) ; 403 | - **server** : a renderPlot (a graph of the data create in a run). 404 | 405 | As a credible usecase, the results of the runs are retrieved and can be displayed. 406 | 407 | ``` r 408 | runApp(system.file("demo_app", package = "shinybatch")) 409 | ``` 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 |
418 | -------------------------------------------------------------------------------- /inst/article/article.md: -------------------------------------------------------------------------------- 1 | # shinybatch 2 | 3 | Dans la famille des packages **shiny** chez **Datastorm**, je demande aujourd'hui le père, à savoir **shinybatch**. **shinybatch** réponds à la problématique suivante : 4 | 5 | Si les utilisateurs *métier* d'une application **shiny** ont besoin de paramétrer, lancer un calcul long à exécuter et pouvant consommant beaucoup de ressources, comment pouvons-nous faire pour leur proposer cela facilement et sans devoir attendre les résultats dans l’application et surcharger le serveur ? 6 | 7 | Effectivement : 8 | 9 | - Si on lance ce calcul directement dans la session **shiny**, l'utilisateur devra attendre la fin de son exécution pour pouvoir continuer à explorer l’application. Et en prime, il risque de bloquer les autres utilisateurs potentiels... 10 | 11 | - On peut aussi utiliser de la programmation asynchrone avec les packages **promise** et **future**. On déporte alors le calcul en dehors de l'application **shiny**, ce qui fait que l'utilisateur peut continuer à travailler en attendant de récupérer son résultat, cela sans bloquer les autres utilisateurs. C'est donc très pertinent pour des calculs qui nécessitent un certain temps (quelques secondes voir minutes). 12 | 13 | Cependant, au-delà de la minute, on doit trouver une alternative. En effet, on imagine mal un utilisateur patientier plusieurs minutes, encore pire plusieurs heures, avec son navigateur ouvert en attendant ses résultats ! 14 | 15 | **shinybatch** a donc été développé pour faciliter la gestion de ces types de calculs, avec : 16 | 17 | - Un module pour configurer une tâche (``configure_task_server``) 18 | - Un batch configurable (heure(s) de lancement, nombre maximal de tâches pouvant être exécuter en même temps) pour lancer les tâches (``scheduler_add``, ``launcher``) 19 | - Un module pour voir l’état des tâches et récupérer les résultats (``tasks_overview_server`` & ``tasks_overview_UI``) 20 | 21 | ### Installation 22 | 23 | Le package est déjà largement utilisé chez **Datastorm** dans nos différents projets. Une *V1* arrivera prochainement sur le **CRAN**. En attendant, vous pouvez l'installer directement depuis notre github https://github.com/datastorm-open : 24 | 25 | ``` r 26 | if(!require(devtools)) install.packages("devtools") 27 | devtools::install_github("datastorm-open/shinybatch") 28 | ``` 29 | 30 | ### Application de démonstration 31 | 32 | Une application de démonstration est disponible 33 | 34 | - directement dans le package : 35 | 36 | ``` r 37 | shiny::runApp(system.file("demo_app", package = "shinybatch")) 38 | ``` 39 | 40 | - et dans la vidéo ci-dessous 41 | 42 | ### Fonctionnement 43 | 44 | #### Fonctions principales 45 | 46 | Concrètement, il suffit d'utiliser les 3 fonctions suivantes : 47 | 48 | - **configure_task_server** : module **shiny** permettant à l'utilisateur de définir une tâche. 49 | - **tasks_overview_server** : module **shiny** pour visualiser les tâches enregistrées, leurs statuts et récupérer les résultats 50 | - **scheduler_add** : Création du script **R** utilisé par le *cron* et définition du *cron*. 51 | 52 | Ces 3 fonctions faisant appel aux autres sous-fonctions disponibles : 53 | 54 | - **configure_task** : Enregistrement des informations de la tâche à effectuer dans un fichier de configuration *.yml* 55 | - **run_task** : Exécution d'une tâche à partir du fichier de configuration *.yml* 56 | - **launcher** : Sélection et lancement de(s) tâche(s), en appelent **run_task** en *batch* avec la commande *RScript* 57 | - **scheduler_init** : Création du script **R** utilisé par le *cron* 58 | 59 | #### Définition d'une tâche 60 | 61 | Une tâche est stockée dans un dossier *parent*, et est définie par un fichier configuration *.yml* contenant les informations suivantes : 62 | 63 | ``` yml 64 | run_info: 65 | date_creation: 2020-04-24 15:21:00 66 | date_start: N/A 67 | date_end: N/A 68 | priority: 1.0 69 | status: waiting 70 | descriptive: 71 | title: my_title 72 | description: my_descr 73 | function: 74 | path: /path/to/my_fun 75 | name: my_fun_name 76 | args: 77 | x: 1.0 78 | 'y': 79 | _path: /path/to/task/dir/inputs/y.RDS 80 | z: 81 | _path: /path/to/task/dir/inputs/z.RDS 82 | dir: /path/to/task/dir/ 83 | ``` 84 | 85 | On retrouve dans la partie ``run_info`` des informations sur la date de création de la tâche, sa priorité et son statut. 86 | 87 | - La priorité est un entier, mis à 0 par défaut. Plus elle est elevé, plus la tâche est prioritaire. 88 | - On dénombre 4 statuts : 89 | + **waiting**, en attente 90 | + **running**, en cours d'exécution 91 | + **finished**, tâche terminée 92 | + **error**, erreur pendant l'exécution 93 | 94 | La partie ``descriptive`` permet de passer des champs descriptifs libres (ici un titre et une description rapide) 95 | 96 | Les informations sur la fonction à lancer sont présentes dans ``function``, avec : 97 | 98 | - **path** : le chemin vers le script **R** principale à sourcer avant de pouvoir lancer le calcul. Ce script doit être autoporteur et contenir toutes les ressources nécessaires (packages, variables, données, sous-fonctions, ...) ainsi que la définition de la fonction principale 99 | 100 | ``` r 101 | # Load package(s) (if needed) 102 | require(data.table) 103 | 104 | # source script(s) (if needed) 105 | source("/path/to/script") 106 | 107 | # Load data (if needed) 108 | data <- readRDS("/path/to/script") 109 | 110 | # Define main function (needed !) 111 | my_fun_name <- function(x, y, z){ 112 | ... 113 | } 114 | ``` 115 | 116 | - **name** : le nom de la fonction principale 117 | 118 | Finalement, les arguments sont stockés dans le champs ``args`` : 119 | 120 | - avec leur valeur directement si possible 121 | - sinon, ils sont enregistrés en *.RDS* dans le répertoire de la tâche (*dir_conf/inputs/arg_name.RDS*) 122 | 123 | Quand une tâche est lancée, les champs ``date_start``, ``date_end`` et ``status`` sont modifiés en conséquence. De plus : 124 | 125 | - un fichier de log est disponible (*dir_conf/output/log_run.txt*) 126 | - le résultat de la fonction est sauvegardé (*dir_conf/output/res.RDS*) 127 | 128 | #### Configuration de lanceur de tâches 129 | 130 | Pour ensuite exécuter les tâches enregistrées, **shinybatch** se base sur un ``cron`` en linux ou un ``Task Scheduler`` en windows. Vous pouvez les créer directement, ou bien utiliser la fonction dédiée **scheduler_add**. La commande par défaut étant : 131 | 132 | ``Rscript /path/to/scheduler_script.R`` 133 | 134 | Le *scheduler_script.R* se compose simplement de l'appel à notre ``launcher()`` avec les paramètres souhaités : 135 | 136 | ```` 137 | #!/usr/bin/env Rscript 138 | args = commandArgs(trailingOnly = TRUE) 139 | 140 | shinybatch::launcher(dir_path = '/path/to/main_directory/', 141 | max_runs = 1, 142 | ignore_status = c('running','finished','error'), 143 | delay_reruns = TRUE 144 | ) 145 | ```` 146 | *N.B :* **scheduler_add** crée par défaut ce script **R** en appelant la fonction **scheduler_init** 147 | 148 | **Fonctionnement du lanceur :** 149 | 150 | La fonction ``launcher()`` regarde toutes les tâches présentes dans le dossier parent. A partir de là : 151 | 152 | - il vérifie que des tâches sont en attente (hors statuts ignorés définis par ``ignore_status``) 153 | - il regarde si des tâches sont en cours d'exécution. Si le nombre maximal de tâches simultanés ``max_runs`` n'est pas atteint, il lance alors des nouvelles tâches : 154 | + avec la priorité maximale ``priority`` 155 | + et la date de création ``date_creation`` la plus ancienne 156 | 157 | ### Modules shiny 158 | 159 | #### Configuration d'une nouvelle tâche 160 | 161 | Le module ``configure_task_server`` se compose uniquement d'une partie *server*, et se branche simplement à un bouton défini en amont et permettant de valider l'enregistrement de la tâche. 162 | 163 | Il prend en entrée l'ensemble des informations nécessaires pour la définir (chemin du script **R** principale, nom de la fonction, arguments, ...) 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | ``` r 172 | ?configure_task_server 173 | 174 | # ui : just create an actionButton 175 | actionButton("go_task", "Configure the task !") 176 | 177 | # server 178 | # call module to configure a task 179 | # connect app inputs to the module 180 | callModule(configure_task_server, "my_id_1", 181 | btn = reactive(input$go_task), 182 | dir_path = dir_conf, 183 | conf_descr = reactive( 184 | list( 185 | "title" = input$title, 186 | "description" = input$description 187 | ) 188 | ), 189 | fun_path = paste0(dir_fun, "/fun_script.R"), 190 | fun_name = "my_fun", 191 | fun_args = reactive( 192 | list( 193 | n = input$fun_nb_points, 194 | mean = input$fun_mean, 195 | sd = input$fun_sd, 196 | sleep = input$sleep_time 197 | ) 198 | ), 199 | priority = reactive(input$priority) 200 | ) 201 | ``` 202 | 203 | 204 | #### Visualisation des tâches enregistrées 205 | 206 | Le module ``tasks_overview`` permet quant à lui de voir l'avancement des tâches enregistrées, et le cas échéant de récupérer les résultats : 207 | 208 | 209 | Il affiche deux tables : 210 | 211 | - une listant toutes les tâches 212 | 213 | 214 | 215 | - et une contenant les informations d'une tâche en particulier, sélectionnée en cliquant sur le premier tableau 216 | 217 | 218 | 219 | Finalement, il retourne côté *server* les informations sur la tâche choisie, ce qui permet donc ensuite de proposer l'affichage / l'exportation des résultats à l'utilisateur ! 220 | 221 | ``` r 222 | ?tasks_overview_UI 223 | 224 | # ui 225 | tasks_overview_UI("my_id_2") 226 | 227 | # server 228 | # call module to view tasks 229 | sel_task <- callModule( 230 | tasks_overview_server, "my_id_2", 231 | dir_path = dir_conf 232 | ) 233 | ``` 234 | -------------------------------------------------------------------------------- /inst/demo_app/global.R: -------------------------------------------------------------------------------- 1 | require(shiny) 2 | require(shinydashboard) 3 | require(DT) 4 | require(shinybatch) 5 | # require(cronR) # decomment for shinyapps.io deployement 6 | # require(markdown) # decomment for shinyapps.io deployement 7 | # CRON not launched on shinyapps... 8 | # create directory for conf 9 | dir_conf <- paste0(tempdir(), "/conf") 10 | if (dir.exists(dir_conf)) unlink(dir_conf, recursive = TRUE) 11 | dir.create(dir_conf, recursive = T) 12 | 13 | # get fun path 14 | fun_path = system.file("ex_fun/sb_fun_ex_demo_app.R", package = "shinybatch") 15 | 16 | # check if cron already existed 17 | # else create it 18 | exists_cron <- scheduler_exist("cr_sc_demo") 19 | if(!exists_cron){ 20 | 21 | # init dir_scheduler 22 | dir_scheduler <- paste0(tempdir(), "/cron") 23 | if (dir.exists(dir_scheduler)) unlink(dir_scheduler, recursive = TRUE) 24 | dir.create(dir_scheduler, recursive = T) 25 | 26 | scheduler_add(dir_scheduler = dir_scheduler, 27 | dir_conf = dir_conf, 28 | max_runs = 1, 29 | head_rows = NULL, 30 | taskname = "cr_sc_demo", 31 | timeout = 0.1) 32 | } 33 | -------------------------------------------------------------------------------- /inst/demo_app/server.R: -------------------------------------------------------------------------------- 1 | # define server 2 | # call both shinybatch modules + display result 3 | server <- function(input, output, session) { 4 | 5 | # call module to configure a task 6 | # connect app inputs to the module 7 | callModule(configure_task_server, "my_id_1", 8 | btn = reactive(input$go_task), 9 | dir_path = dir_conf, 10 | conf_descr = reactive(list("title" = input$title, 11 | "description" = input$description)), 12 | fun_path = fun_path, 13 | fun_name = "my_fun", 14 | fun_args = reactive(list(n = input$fun_nb_points, 15 | mean = input$fun_mean, 16 | sd = input$fun_sd, 17 | sleep = input$sleep_time)), 18 | priority = reactive(input$priority)) 19 | 20 | # call module to view tasks 21 | sel_task <- callModule(tasks_overview_server, "my_id_2", 22 | dir_path = dir_conf, 23 | update_mode = "reactive", 24 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 25 | allowed_run_info_cols = NULL, 26 | allowed_function_cols = NULL, 27 | allow_log_btn = T, 28 | allow_rm_task = T, 29 | allow_descr = T, 30 | allow_args = T) 31 | 32 | # check if sel_task can be displayed 33 | output$launch_task <- reactive({ 34 | ! is.null(sel_task()) && length(sel_task()) > 0 && sel_task()$status == "finished" 35 | }) 36 | outputOptions(output, "launch_task", suspendWhenHidden = FALSE) 37 | 38 | # display task 39 | output$task_plot <- renderPlot({ 40 | cpt <- input$show_task 41 | sel_task <- sel_task() 42 | 43 | isolate({ 44 | if (cpt > 0 && length(sel_task) > 0 && sel_task$status == "finished") { 45 | 46 | data <- readRDS(paste0(sel_task$path, "/res.RDS")) 47 | 48 | plot(density(data), col = "#3c8dbc", lwd = 2, main = "Density plot of generated observations", xlab = "", ylab = "Density", axes = F) 49 | axis(1) 50 | axis(2) 51 | } 52 | }) 53 | }) 54 | 55 | onStop(function() { 56 | scheduler_remove(taskname = "cr_sc_demo") 57 | }) 58 | } 59 | -------------------------------------------------------------------------------- /inst/demo_app/ui.R: -------------------------------------------------------------------------------- 1 | # Define UI for application that draws a histogram 2 | navbarPage(title = HTML(paste0('

', paste0(rep(" ", 25), collapse = ""), ' shinybatch

')), id = "nav-id", collapsible = TRUE, 3 | position = "fixed-top", theme = "css/custom.css", 4 | header = div( 5 | br(), br(), br(), br(), 6 | a(href = "https://www.datastorm.fr", 7 | target = "_blank", img(src = "img/img-datastorm-logo-white.png", class = "ribbon", style = "margin-left: 0cm;margin-top: 0.1cm;height: 55px")), 8 | a(href = "https://github.com/datastorm-open/shinybatch", 9 | target = "_blank", img(src = "img/github.png", class = "ribbon", style = "margin-left: 3cm;margin-top: 0cm;height: 60px")), 10 | # footer 11 | div(class = "ds_app_footer", div(p("copyright © Datastorm 2020", style = "color:white"), align = "center")), 12 | ), 13 | windowTitle = "shinybatch", 14 | tabPanel("Introduction", 15 | includeMarkdown("www/script/intro.md") 16 | ), 17 | tabPanel("Démo", 18 | fluidRow( 19 | shinydashboard::tabBox(width = 12, 20 | tabPanel("Configure tasks", 21 | fluidRow( 22 | column(2, 23 | numericInput("fun_nb_points", "Number of points", 24 | min = 1, max = 10000, value = 100, step = 1) 25 | ), 26 | column(2, 27 | sliderInput("fun_mean", "Mean", 28 | min = -10, max = 10, value = 0, step = 0.1, width = "100%") 29 | ), 30 | column(2, 31 | sliderInput("fun_sd", "Standard deviation", 32 | min = 0, max = 100, value = 1, step = 0.5, width = "100%") 33 | ), 34 | column(2, 35 | numericInput("sleep_time", "Sleep time (s)", 36 | min = 0, max = 30, value = 0, step = 1) 37 | ) 38 | ), 39 | fluidRow( 40 | column(2, 41 | numericInput("priority", "Priority", 42 | min = 0, max = 10, value = 1, step = 1) 43 | ), 44 | column(3, 45 | textInput("title", "Title", 46 | value = "Task title", width = "100%") 47 | ), 48 | column(3, 49 | textInput("description", "Description", 50 | value = "Task description", width = "100%") 51 | ) 52 | ), 53 | hr(), 54 | fluidRow( 55 | column(6, offset = 3, 56 | actionButton("go_task", "Configure the task", width = "100%") 57 | ) 58 | ) 59 | 60 | ), 61 | tabPanel("View tasks", 62 | fluidRow( 63 | column(12, 64 | tasks_overview_UI("my_id_2") 65 | ) 66 | ), 67 | conditionalPanel(condition = "output.launch_task", 68 | br(), 69 | hr(), 70 | fluidRow( 71 | conditionalPanel(condition = "output.launch_task", 72 | column(12, 73 | div(actionButton("show_task", "Display task result", width = "40%"), 74 | align = "center") 75 | ) 76 | ), 77 | column(12, 78 | conditionalPanel(condition = "input.show_task > 0", 79 | plotOutput("task_plot") 80 | ) 81 | ) 82 | ) 83 | ) 84 | ) 85 | ) 86 | ) 87 | ), 88 | tabPanel("Code", 89 | fluidRow( 90 | shinydashboard::tabBox(width = 12, 91 | tabPanel("global.R", 92 | includeMarkdown("www/script/global.md") 93 | ), 94 | tabPanel("server.R", 95 | includeMarkdown("www/script/server.md") 96 | ), 97 | tabPanel("ui.R", 98 | includeMarkdown("www/script/ui.md") 99 | ) 100 | ) 101 | ) 102 | ), 103 | br(), br(), br() 104 | ) 105 | -------------------------------------------------------------------------------- /inst/demo_app/www/css/custom.css: -------------------------------------------------------------------------------- 1 | .selectize-control.multi .selectize-input:after { 2 | content: ''; 3 | display: block; 4 | position: absolute; 5 | top: 50%; 6 | right: 17px; 7 | margin-top: -3px; 8 | width: 0; 9 | height: 0; 10 | border-style: solid; 11 | border-width: 5px 5px 0 5px; 12 | border-color: #333333 transparent transparent transparent; 13 | } 14 | 15 | .ds_app_footer { 16 | position: fixed; 17 | left: 0; 18 | bottom: 0; 19 | width: 100%; 20 | /*background-color: #CC0000; 21 | border-color: #AA0000;*/ 22 | background-color: #430838; 23 | border-color: #0dc6ff; 24 | border-radius: 0; 25 | /*background-image: url('../img/bp_background.png');*/ 26 | color: white; 27 | text-align: center; 28 | z-index: 99999; 29 | } 30 | 31 | 32 | img[alt="img"] { 33 | width: 90%; 34 | display: block; 35 | margin-left: auto; 36 | margin-right: auto 37 | } 38 | 39 | .modal.fade.in { 40 | z-index: 100000 !important; 41 | } 42 | 43 | .navbar-default { 44 | /*background-color: #CC0000; 45 | border-color: #AA0000; 46 | border-radius: 0; 47 | background-image: url('../img/bp_background.png');*/ 48 | background-color: #430838; 49 | z-index: 99999; 50 | display: inline-block; 51 | } 52 | 53 | 54 | .navbar-default .navbar-brand, 55 | .navbar-default .navbar-brand:hover, 56 | .navbar-default .navbar-brand:focus { 57 | display: inline-block; 58 | color: #FFF; 59 | padding-top:20px !important; 60 | padding-bottom:20px !important; 61 | font-size: 1.6em !important; 62 | } 63 | 64 | .navbar-default .navbar-nav > li > a { 65 | display: inline-block; 66 | color:#FFF; 67 | opacity:.5; 68 | font-weight:500; 69 | padding-top:20px !important; 70 | padding-bottom:20px !important; 71 | font-size: 1.35em !important; 72 | } 73 | 74 | .navbar-default .navbar-nav > li > a:hover { 75 | display: inline-block; 76 | text-decoration:none; 77 | color:#FFF; 78 | opacity:1 79 | } 80 | 81 | .navbar-default .navbar-nav > li > a:focus { 82 | background-color: #430838; 83 | border-bottom:5px solid #e6e6e6; 84 | display: inline-block; 85 | color:#FFF; 86 | opacity:1 87 | } 88 | 89 | .navbar-default .navbar-nav > .active > a, 90 | .navbar-default .navbar-nav > .active > a:hover, 91 | .navbar-default .navbar-nav > .active > a:focus { 92 | background-color: #430838; 93 | border-bottom:5px solid #e6e6e6; 94 | color:#FFF; 95 | opacity:1 96 | } 97 | 98 | .navbar-default .navbar-text { 99 | color: #FFF; 100 | } 101 | 102 | .navbar-default .navbar-toggle { 103 | border-color: #AA0000; 104 | } 105 | 106 | .navbar-default .navbar-toggle:hover, 107 | .navbar-default .navbar-toggle:focus { 108 | background-color: #AA0000; 109 | } 110 | 111 | .navbar-default .navbar-toggle .icon-bar { 112 | background-color: #FFF; 113 | } 114 | 115 | 116 | .ribbon { 117 | position: fixed; 118 | left: 0; 119 | top: 0; 120 | z-index: 100000; 121 | margin-left: 0cm; 122 | margin-top: 0.1cm; 123 | display: block; 124 | height: 65px; 125 | text-decoration: none; 126 | overflow-x: hidden; 127 | } 128 | 129 | body,p,html{ 130 | font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif; 131 | } 132 | 133 | h1,h2,h3,h4,h5,h6{ 134 | font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif; 135 | color: #430838; 136 | /*font-weight: bold*/ 137 | } 138 | 139 | a{ 140 | color: black; 141 | } 142 | a:hover { 143 | color: #e6e6e6; 144 | } 145 | -------------------------------------------------------------------------------- /inst/demo_app/www/figures/conf_file.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/conf_file.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/custom_cron.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/custom_cron.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/default_cron.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/default_cron.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/demo_app_conf_task.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/demo_app_conf_task.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/demo_app_view_task_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/demo_app_view_task_1.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/demo_app_view_task_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/demo_app_view_task_2.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/launch_task_shiny.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/launch_task_shiny.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/launch_task_shiny_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/launch_task_shiny_2.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/launch_task_shiny_code.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/launch_task_shiny_code.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/see_tasks_shiny.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/see_tasks_shiny.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/see_tasks_shiny_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/see_tasks_shiny_2.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/figures/see_tasks_shiny_code.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/figures/see_tasks_shiny_code.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/img/github.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/img/github.png -------------------------------------------------------------------------------- /inst/demo_app/www/img/img-datastorm-logo-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/img/img-datastorm-logo-white.png -------------------------------------------------------------------------------- /inst/demo_app/www/img/logoDS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/img/logoDS.png -------------------------------------------------------------------------------- /inst/demo_app/www/img/rpivot.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app/www/img/rpivot.PNG -------------------------------------------------------------------------------- /inst/demo_app/www/script/global.md: -------------------------------------------------------------------------------- 1 | ``` r 2 | require(shiny) 3 | require(shinydashboard) 4 | require(DT) 5 | require(shinybatch) 6 | 7 | # create directory for conf 8 | dir_conf <- paste0(tempdir(), "/conf") 9 | if (dir.exists(dir_conf)) unlink(dir_conf, recursive = TRUE) 10 | dir.create(dir_conf, recursive = T) 11 | 12 | # get fun path 13 | fun_path = system.file("ex_fun/sb_fun_ex_demo_app.R", package = "shinybatch") 14 | 15 | # check if cron already existed 16 | # else create it 17 | exists_cron <- scheduler_exist("cr_sc_demo") 18 | if(!exists_cron){ 19 | 20 | # init dir_scheduler 21 | dir_scheduler <- paste0(tempdir(), "/cron") 22 | if (dir.exists(dir_scheduler)) unlink(dir_scheduler, recursive = TRUE) 23 | dir.create(dir_scheduler, recursive = T) 24 | 25 | scheduler_add(dir_scheduler = dir_scheduler, 26 | dir_conf = dir_conf, 27 | max_runs = 1, 28 | head_rows = NULL, 29 | taskname = "cr_sc_demo") 30 | } 31 | ``` 32 | -------------------------------------------------------------------------------- /inst/demo_app/www/script/intro.md: -------------------------------------------------------------------------------- 1 | # shinybatch 2 | 3 | This package provides a simple framework to create, launch *automatically* and retrieve *time-consuming operations* (tasks) **in batch mode** from a **Shiny app**. 4 | 5 | The tasks are automatically launched using a scheduler, e.g. a timer that periodically launches a (batch) operation. 6 | 7 | - with a CRON in Linux/Mac, using package **cronR** 8 | - with Windows Task Scheduler in Windows, using package **taskscheduleR** 9 | - or defined directly with OS tools 10 | 11 | ### Installation 12 | 13 | ``` r 14 | devtools::install_github("datastorm-open/shinybatch") 15 | ``` 16 | 17 | ### Main functions 18 | 19 | - **configure_task** : create a *.yml* file filed with operation params (fun path, fun args, priority, ...), 20 | - **run_task** : run a selected task from a *.yml* file. Used by **launcher**, 21 | - **launcher** : select and run the task(s) with highest priority, calling **run_task** in batch mode with *RScript* command, 22 | - **scheduler_init** : (opt: create the cron R script) create the file to be launched by the *scheduler*, 23 | - **scheduler_add** : (opt: create the cron file, and by default the cron R script) and start the *scheduler* which will read the file at the given frequency to launch batch operations, 24 | - **configure_task_server** : define a task and call *configure_task()* in the Shiny app, 25 | - **tasks_overview_server** : display tasks & retrieve results within the Shiny app. 26 | 27 | 28 | ### Definition of a task 29 | 30 | *configure_task()* 31 | 32 | A task is defined by its *.yml* file that contains the following informations : 33 | 34 | ``` yml 35 | run_info: 36 | date_creation: 2020-04-24 15:21:00 37 | date_start: N/A 38 | date_end: N/A 39 | priority: 1.0 40 | status: waiting 41 | descriptive: 42 | title: my_title 43 | description: my_descr 44 | function: 45 | path: /path/to/my_fun 46 | name: my_fun_name 47 | args: 48 | x: 1.0 49 | 'y': 50 | _path: /path/to/task/dir/inputs/y.RDS 51 | z: 52 | _path: /path/to/task/dir/inputs/z.RDS 53 | dir: /path/to/task/dir/ 54 | ``` 55 | 56 | 57 | The ``run_info`` part contains general informations about the task. 58 | 59 | Priority can be any number, with 0 as default. The highest the priority, the sooner it is launched. 60 | 61 | Valid status are: 62 | 63 | - **waiting** 64 | - **running** 65 | - **finished** 66 | - **error** 67 | 68 | The ``descriptive`` part contains free informations given by the user. The title and description fields are only example. 69 | 70 | The ``function`` part contains the location of the fun **R** script (for sourcing) and its name (for calling). The script must have all necessary ressources (packages, variables, functions) for execute the main function : 71 | 72 | ``` r 73 | # Load package(s) (if needed) 74 | require(data.table) 75 | 76 | # source script(s) (if needed) 77 | source("/path/to/script") 78 | 79 | # Load data (if needed) 80 | data <- readRDS("/path/to/script") 81 | 82 | # Define main function (needed !) 83 | my_fun_name <- function(x, y, z){ 84 | ... 85 | } 86 | ``` 87 | 88 | The ``args`` part contains either the argument itself if it is of length 1 or the location of the argument (in *dir_conf/inputs/arg_name.RDS*), complex arguments are storing as *.RDS* 89 | 90 | The ``dir`` argument contains the location of the directory in which is stored the *conf.yml* file. 91 | 92 | When a task has been succesfully run, some fields are updated: 93 | 94 | - ``date_start`` and ``date_end`` are filled, 95 | - ``status`` is set to 'running', then to 'finished'. 96 | 97 | Some outputs are created: 98 | 99 | - if wanted, the result of the task, in other words the output of the main function (in *dir_conf/output/res.RDS*) 100 | - the log of the run (in *dir_conf/output/log_run.txt*) 101 | 102 | ### Description of the launcher 103 | 104 | *launcher()* 105 | 106 | The launcher retrieves all the tasks in a main directory and build a the table of their *run_info*. Based on this, it verifies that there are tasks with *status* that allow a run, e.g. all but those in *ignore_status*. Then, if the maximum number of simultaneously running tasks is not reached, it launches new tasks according to their priority, in batch mode with *RScript* command (and calling *run_task()*) 107 | 108 | 109 | The task with higher priority is defined as the one: 110 | 111 | - with *status* not in *ignore_status* (default is all but **waiting**), 112 | - which has the highest *priority*, 113 | - and then the oldest *date_creation*. 114 | 115 | ### Description of the scheduler 116 | 117 | Before calling the scheduler, we first need to create the file that it will launch. We can use **scheduler_init()**. By default, it looks like this (*scheduler_script.R*) : 118 | 119 | ```` 120 | #!/usr/bin/env Rscript 121 | args = commandArgs(trailingOnly = TRUE) 122 | 123 | shinybatch::launcher(dir_path = '/path/to/main_directory/', 124 | max_runs = 1, 125 | ignore_status = c('running','finished','error'), 126 | delay_reruns = TRUE 127 | ) 128 | ```` 129 | 130 | ... but the head lines can be customized by filling the *head_rows* argument. 131 | 132 | 133 | Once the file has been created, the cron has to be defined. You can use directly ``cron`` on linux or the ``Task Scheduler`` on windows, or the **scheduler_add** function. The default command is : 134 | 135 | ``Rscript /path/to/scheduler_script.R`` 136 | 137 | *N.B :* **scheduler_add** creates by default the **R** scheduler script using **scheduler_init** 138 | 139 | ### Example 140 | 141 | **sb_fun_ex.R** 142 | 143 | ``` r 144 | sb_fun_ex <- function(x, y, z) { 145 | res <- x + y 146 | message('Running !') 147 | warning("Complex (or not) variable z is not used...!", call. = FALSE) 148 | res 149 | } 150 | ``` 151 | 152 | **Configure a task** 153 | ``` r 154 | require(shinybatch) 155 | 156 | ?configure_task 157 | 158 | # create temporary directory 159 | dir <- tempdir() 160 | 161 | # create and save conf 162 | conf <- configure_task( 163 | dir_path = dir, 164 | conf_descr = list( 165 | title = "my_title", 166 | description = "my_descr" 167 | ), 168 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch"), # as an example, 169 | fun_name = "sb_fun_ex", 170 | fun_args = list( 171 | x = 1, 172 | y = 0:4, 173 | z = iris 174 | ), 175 | priority = 1) 176 | 177 | # check results 178 | list.files(conf$dir, recursive = T) 179 | # [1] "conf.yml" "inputs/y.RDS" "inputs/z.RDS" 180 | 181 | read_conf <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 182 | 183 | read_conf$run_info 184 | # $date_creation 185 | # [1] "2021-03-02 16:54:24" 186 | # 187 | # $date_start 188 | # [1] "N/A" 189 | # 190 | # $date_end 191 | # [1] "N/A" 192 | # 193 | # $priority 194 | # [1] 1 195 | # 196 | # $status 197 | # [1] "waiting" 198 | 199 | read_conf$args$x 200 | # [1] 1 201 | 202 | y <- readRDS(paste0(conf$dir, "inputs/y.RDS")) 203 | # [1] 0 1 2 3 4 204 | 205 | z <- readRDS(paste0(conf$dir, "inputs/z.RDS")) 206 | ``` 207 | 208 | **Run one given task (for demo/test)** 209 | 210 | ``` r 211 | ?run_task 212 | 213 | run_task(paste0(conf$dir, "conf.yml")) 214 | 215 | # catch results 216 | list.files(conf$dir, recursive = T) 217 | # output directory with log & result 218 | # [1] "conf.yml" "inputs/y.RDS" 219 | # [3] "inputs/z.RDS" "output/log_run_20210302_1656_1080.txt" 220 | # [5] "output/res.RDS" 221 | 222 | conf_update <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 223 | conf_update$run_info 224 | # $date_creation 225 | # [1] "2021-03-02 16:54:24" 226 | # 227 | # $date_start 228 | # [1] "2021-03-02 16:56:10" 229 | # 230 | # $date_end 231 | # [1] "2021-03-02 16:56:10" 232 | # 233 | # $priority 234 | # [1] 1 235 | # 236 | # $status 237 | # [1] "finished" 238 | 239 | 240 | output <- readRDS(paste0(conf$dir, "output/res.RDS")) 241 | #[1] 1 2 3 4 5 242 | 243 | log <- read.delim(list.files(paste0(conf$dir, "output/"), pattern = "log_run", full.names = T), header = F) 244 | # [2021-03-02 16:56:10] [INFO] Starting task execution... 245 | # [2021-03-02 16:56:10] [INFO] Running ! 246 | # [2021-03-02 16:56:10] [WARN] Complex (or not) variable z is not used...! 247 | # [2021-03-02 16:56:10] [INFO] ... task terminated. 248 | ``` 249 | 250 | **Use scheduler to launch the tasks** 251 | 252 | ``` r 253 | ?scheduler_add 254 | 255 | # create temporary directory for conf 256 | dir_conf <- paste0(tempdir(), "/conf/") 257 | dir.create(dir_conf, recursive = T) 258 | 259 | # create 2 confs 260 | conf_1 <- configure_task( 261 | dir_path = dir_conf, 262 | conf_descr = list( 263 | title_1 = "my_title_1", 264 | description_1 = "my_descr_1" 265 | ), 266 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch"), # as an example, 267 | fun_name = "sb_fun_ex", 268 | fun_args = list( 269 | x = 0, 270 | y = 0:4, 271 | z = iris 272 | ), 273 | priority = 1) 274 | 275 | conf_2 <- configure_task( 276 | dir_path = dir_conf, 277 | conf_descr = list( 278 | title_1 = "my_title_2", 279 | description_1 = "my_descr_2" 280 | ), 281 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch"), # as an example, 282 | fun_name = "sb_fun_ex", 283 | fun_args = list( 284 | x = 0, 285 | y = 0:4, 286 | z = iris 287 | ), 288 | priority = 2) 289 | 290 | # on LINUX -> Needs cronR package 291 | # on Windows -> Needs taskscheduleR package 292 | 293 | scheduler_add( 294 | dir_scheduler = tempdir(), 295 | dir_conf = dir_conf, 296 | max_runs = 1, 297 | create_file = T, 298 | head_rows = NULL, 299 | taskname = "cron_script_ex" 300 | ) 301 | 302 | scheduler_ls() # display existing crons 303 | 304 | # wait up to 1 min for conf_2 and up to 2 mins for conf_1 305 | yaml::read_yaml(paste0(conf_1$dir, "/conf.yml"))$run_info$status 306 | yaml::read_yaml(paste0(conf_2$dir, "/conf.yml"))$run_info$status 307 | 308 | scheduler_rm(id = "cron_script_ex") # kill selected cron 309 | ``` 310 | 311 | **Shiny modules** 312 | 313 | These modules contain the basic framework to use all the previous functions in a Shiny app. 314 | Both are used in the demo app which presents a simple usecase. 315 | 316 | - **Configure a new task** 317 | 318 | ![img](figures/launch_task_shiny.PNG) 319 | 320 | ![img](figures/launch_task_shiny_2.PNG) 321 | 322 | call: 323 | 324 | ``` r 325 | ?configure_task_server 326 | 327 | # ui : just create an actionButton 328 | actionButton("go_task", "Configure the task !") 329 | 330 | # server 331 | # call module to configure a task 332 | # connect app inputs to the module 333 | callModule(configure_task_server, "my_id_1", 334 | btn = reactive(input$go_task), 335 | dir_path = dir_conf, 336 | conf_descr = reactive( 337 | list( 338 | "title" = input$title, 339 | "description" = input$description 340 | ) 341 | ), 342 | fun_path = paste0(dir_fun, "/fun_script.R"), 343 | fun_name = "my_fun", 344 | fun_args = reactive( 345 | list( 346 | n = input$fun_nb_points, 347 | mean = input$fun_mean, 348 | sd = input$fun_sd, 349 | sleep = input$sleep_time 350 | ) 351 | ), 352 | priority = reactive(input$priority) 353 | ) 354 | ``` 355 | 356 | 357 | - **Display configured tasks** 358 | 359 | ![img](figures/see_tasks_shiny.PNG) 360 | 361 | ![img](figures/see_tasks_shiny_2.PNG) 362 | 363 | 364 | call: 365 | 366 | ``` r 367 | ?tasks_overview_UI 368 | 369 | # ui 370 | tasks_overview_UI("my_id_2") 371 | 372 | # server 373 | # call module to view tasks 374 | sel_task <- callModule( 375 | tasks_overview_server, "my_id_2", 376 | dir_path = dir_conf, 377 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 378 | allowed_run_info_cols = NULL, 379 | allowed_function_cols = NULL, 380 | allow_descr = T, 381 | allow_args = T 382 | ) 383 | ``` 384 | 385 | This module returns : 386 | 387 | - the status of the selected line (one run) of the summary table, 388 | - the path to the directory in which its output is stored. 389 | 390 | Thus we know when a run is finished and we can load its result to reuse/display it : (readRDS(paste0(path, "/res.RDS"))). 391 | 392 | **Demo app** 393 | 394 | A demo app to create and automatically launch an example task : the generation of normally distributed observations. 395 | 396 | - **global** : the path to the confs directory, the path to the script of the function to be run, the call to scheduler_add() ; 397 | - **ui** : shiny inputs (description args for the conf ; parameters for the function to be called by the cron) ; 398 | - **server** : a renderPlot (a graph of the data create in a run). 399 | 400 | As a credible usecase, the results of the runs are retrieved and can be displayed. 401 | 402 | ``` r 403 | runApp(system.file("demo_app", package = "shinybatch")) 404 | ``` 405 | 406 | ![img](figures/demo_app_conf_task.PNG) 407 | 408 | ![img](figures/demo_app_view_task_1.PNG) 409 | 410 | ![img](figures/demo_app_view_task_2.PNG) 411 | 412 |
413 | -------------------------------------------------------------------------------- /inst/demo_app/www/script/server.md: -------------------------------------------------------------------------------- 1 | ``` r 2 | # call module to configure a task 3 | # connect app inputs to the module 4 | callModule(configure_task_server, "my_id_1", 5 | btn = reactive(input$go_task), 6 | dir_path = dir_conf, 7 | conf_descr = reactive(list("title" = input$title, 8 | "description" = input$description)), 9 | fun_path = fun_path, 10 | fun_name = "my_fun", 11 | fun_args = reactive(list(n = input$fun_nb_points, 12 | mean = input$fun_mean, 13 | sd = input$fun_sd, 14 | sleep = input$sleep_time)), 15 | priority = reactive(input$priority)) 16 | 17 | # call module to view tasks 18 | sel_task <- callModule(tasks_overview_server, "my_id_2", 19 | dir_path = dir_conf, 20 | update_mode = "reactive", 21 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 22 | allowed_run_info_cols = NULL, 23 | allowed_function_cols = NULL, 24 | allow_descr = T, 25 | allow_args = T) 26 | 27 | # check if sel_task can be displayed 28 | output$launch_task <- reactive({ 29 | ! is.null(sel_task()) && length(sel_task()) > 0 && sel_task()$status == "finished" 30 | }) 31 | outputOptions(output, "launch_task", suspendWhenHidden = FALSE) 32 | 33 | # display task 34 | output$task_plot <- renderPlot({ 35 | cpt <- input$show_task 36 | sel_task <- sel_task() 37 | 38 | isolate({ 39 | if (cpt > 0 && length(sel_task) > 0 && sel_task$status == "finished") { 40 | 41 | data <- readRDS(paste0(sel_task$path, "/res.RDS")) 42 | 43 | plot(density(data), col = "#3c8dbc", lwd = 2, 44 | main = "Density plot of generated observations", xlab = "", 45 | ylab = "Density", axes = F) 46 | axis(1) 47 | axis(2) 48 | } 49 | }) 50 | }) 51 | ``` 52 | -------------------------------------------------------------------------------- /inst/demo_app/www/script/ui.md: -------------------------------------------------------------------------------- 1 | ``` r 2 | fluidRow( 3 | tabBox(width = 12, 4 | tabPanel("Configure tasks", 5 | fluidRow( 6 | column(2, 7 | numericInput("fun_nb_points", "Number of points", 8 | min = 1, max = 10000, value = 100, step = 1) 9 | ), 10 | column(2, 11 | sliderInput("fun_mean", "Mean", 12 | min = -10, max = 10, value = 0, step = 0.1, width = "100%") 13 | ), 14 | column(2, 15 | sliderInput("fun_sd", "Standard deviation", 16 | min = 0, max = 100, value = 1, step = 0.5, width = "100%") 17 | ), 18 | column(2, 19 | numericInput("sleep_time", "Sleep time (s)", 20 | min = 0, max = 30, value = 0, step = 1) 21 | ) 22 | ), 23 | fluidRow( 24 | column(2, 25 | numericInput("priority", "Priority", 26 | min = 0, max = 10, value = 1, step = 1) 27 | ), 28 | column(3, 29 | textInput("title", "Title", 30 | value = "Task title", width = "100%") 31 | ), 32 | column(3, 33 | textInput("description", "Description", 34 | value = "Task description", width = "100%") 35 | ) 36 | ), 37 | hr(), 38 | fluidRow( 39 | column(6, offset = 3, 40 | actionButton("go_task", "Configure the task", width = "100%") 41 | ) 42 | ) 43 | 44 | ), 45 | tabPanel("View tasks", 46 | fluidRow( 47 | column(12, 48 | tasks_overview_UI("my_id_2") 49 | ) 50 | ), 51 | conditionalPanel(condition = "output.launch_task", 52 | br(), 53 | hr(), 54 | fluidRow( 55 | conditionalPanel(condition = "output.launch_task", 56 | column(12, 57 | div(actionButton("show_task", "Display task result", width = "40%"), 58 | align = "center") 59 | ) 60 | ), 61 | column(12, 62 | conditionalPanel(condition = "input.show_task > 0", 63 | plotOutput("task_plot") 64 | ) 65 | ) 66 | ) 67 | ) 68 | ) 69 | ) 70 | ) 71 | ``` 72 | -------------------------------------------------------------------------------- /inst/demo_app_conc/global.R: -------------------------------------------------------------------------------- 1 | require(shiny) 2 | require(shinydashboard) 3 | require(DT) 4 | require(shinybatch) 5 | # require(cronR) # decomment for shinyapps.io deployement 6 | # require(markdown) # decomment for shinyapps.io deployement 7 | # CRON not launched on shinyapps... 8 | # create directory for conf 9 | dir_conf_1 <- paste0(tempdir(), "/conf_1") 10 | if (dir.exists(dir_conf_1)) unlink(dir_conf_1, recursive = TRUE) 11 | dir.create(dir_conf_1, recursive = T) 12 | 13 | dir_conf_2 <- paste0(tempdir(), "/conf_2") 14 | if (dir.exists(dir_conf_2)) unlink(dir_conf_2, recursive = TRUE) 15 | dir.create(dir_conf_2, recursive = T) 16 | 17 | # get fun path 18 | fun_path = system.file("ex_fun/sb_fun_ex_demo_app.R", package = "shinybatch") 19 | 20 | # check if cron already existed 21 | # else create it 22 | exists_cron <- scheduler_exist("cr_sc_demo") 23 | if(!exists_cron){ 24 | 25 | # init dir_scheduler 26 | dir_scheduler <- paste0(tempdir(), "/cron") 27 | if (dir.exists(dir_scheduler)) unlink(dir_scheduler, recursive = TRUE) 28 | dir.create(dir_scheduler, recursive = T) 29 | 30 | scheduler_add(dir_scheduler = dir_scheduler, 31 | dir_conf = c(dir_conf_1, dir_conf_2), 32 | max_runs = 1, 33 | head_rows = NULL, 34 | taskname = "cr_sc_demo") 35 | } 36 | -------------------------------------------------------------------------------- /inst/demo_app_conc/server.R: -------------------------------------------------------------------------------- 1 | # define server 2 | # call both shinybatch modules + display result 3 | server <- function(input, output, session) { 4 | 5 | # call module to configure a task 6 | # connect app inputs to the module 7 | callModule(configure_task_server, "my_id_1", 8 | btn = reactive(input$go_task), 9 | dir_path = dir_conf_1, 10 | conf_descr = reactive(list("title" = input$title, 11 | "description" = input$description)), 12 | fun_path = fun_path, 13 | fun_name = "my_fun", 14 | fun_args = reactive(list(n = input$fun_nb_points, 15 | mean = input$fun_mean, 16 | sd = input$fun_sd, 17 | sleep = input$sleep_time)), 18 | priority = reactive(input$priority)) 19 | 20 | # call module to view tasks 21 | sel_task <- callModule(tasks_overview_server, "my_id_2", 22 | dir_path = dir_conf_1, 23 | update_mode = "reactive", 24 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 25 | allowed_run_info_cols = NULL, 26 | allowed_function_cols = NULL, 27 | allow_log_btn = T, 28 | allow_rm_task = T, 29 | allow_descr = T, 30 | allow_args = T) 31 | 32 | callModule(configure_task_server, "my_id_1_2", 33 | btn = reactive(input$go_task_2), 34 | dir_path = dir_conf_2, 35 | conf_descr = reactive(list("title" = input$title_2, 36 | "description" = input$description_2)), 37 | fun_path = fun_path, 38 | fun_name = "my_fun", 39 | fun_args = reactive(list(n = input$fun_nb_points_2, 40 | mean = input$fun_mean_2, 41 | sd = input$fun_sd_2, 42 | sleep = input$sleep_time_2)), 43 | priority = reactive(input$priority_2)) 44 | 45 | # call module to view tasks 46 | sel_task_2 <- callModule(tasks_overview_server, "my_id_2_2", 47 | dir_path = dir_conf_2, 48 | update_mode = "reactive", 49 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 50 | allowed_run_info_cols = NULL, 51 | allowed_function_cols = NULL, 52 | allow_log_btn = T, 53 | allow_rm_task = T, 54 | allow_descr = T, 55 | allow_args = T) 56 | 57 | # check if sel_task can be displayed 58 | output$launch_task <- reactive({ 59 | ! is.null(sel_task()) && length(sel_task()) > 0 && sel_task()$status == "finished" 60 | }) 61 | outputOptions(output, "launch_task", suspendWhenHidden = FALSE) 62 | 63 | output$launch_task_2 <- reactive({ 64 | ! is.null(sel_task_2()) && length(sel_task_2()) > 0 && sel_task_2()$status == "finished" 65 | }) 66 | outputOptions(output, "launch_task_2", suspendWhenHidden = FALSE) 67 | 68 | # display task 69 | output$task_plot <- renderPlot({ 70 | cpt <- input$show_task 71 | sel_task <- sel_task() 72 | 73 | isolate({ 74 | if (cpt > 0 && length(sel_task) > 0 && sel_task$status == "finished") { 75 | 76 | data <- readRDS(paste0(sel_task$path, "/res.RDS")) 77 | 78 | plot(density(data), col = "#3c8dbc", lwd = 2, main = "Density plot of generated observations", xlab = "", ylab = "Density", axes = F) 79 | axis(1) 80 | axis(2) 81 | } 82 | }) 83 | }) 84 | 85 | output$task_plot_2 <- renderPlot({ 86 | cpt <- input$show_task_2 87 | sel_task <- sel_task_2() 88 | 89 | isolate({ 90 | if (cpt > 0 && length(sel_task) > 0 && sel_task$status == "finished") { 91 | 92 | data <- readRDS(paste0(sel_task$path, "/res.RDS")) 93 | 94 | plot(density(data), col = "#3c8dbc", lwd = 2, main = "Density plot of generated observations", xlab = "", ylab = "Density", axes = F) 95 | axis(1) 96 | axis(2) 97 | } 98 | }) 99 | }) 100 | onStop(function() { 101 | scheduler_remove(taskname = "cr_sc_demo") 102 | }) 103 | } 104 | -------------------------------------------------------------------------------- /inst/demo_app_conc/ui.R: -------------------------------------------------------------------------------- 1 | # Define UI for application that draws a histogram 2 | navbarPage(title = HTML(paste0('

', paste0(rep(" ", 25), collapse = ""), ' shinybatch

')), id = "nav-id", collapsible = TRUE, 3 | position = "fixed-top", theme = "css/custom.css", 4 | header = div( 5 | br(), br(), br(), br(), 6 | a(href = "https://www.datastorm.fr", 7 | target = "_blank", img(src = "img/img-datastorm-logo-white.png", class = "ribbon", style = "margin-left: 0cm;margin-top: 0.1cm;height: 55px")), 8 | a(href = "https://github.com/datastorm-open/shinybatch", 9 | target = "_blank", img(src = "img/github.png", class = "ribbon", style = "margin-left: 3cm;margin-top: 0cm;height: 60px")), 10 | # footer 11 | div(class = "ds_app_footer", div(p("copyright © Datastorm 2020", style = "color:white"), align = "center")), 12 | ), 13 | windowTitle = "shinybatch", 14 | tabPanel("Introduction", 15 | includeMarkdown("www/script/intro.md") 16 | ), 17 | tabPanel("Démo", 18 | fluidRow( 19 | shinydashboard::tabBox(width = 12, 20 | tabPanel("Configure tasks n°1", 21 | fluidRow( 22 | column(2, 23 | numericInput("fun_nb_points", "Number of points", 24 | min = 1, max = 10000, value = 100, step = 1) 25 | ), 26 | column(2, 27 | sliderInput("fun_mean", "Mean", 28 | min = -10, max = 10, value = 0, step = 0.1, width = "100%") 29 | ), 30 | column(2, 31 | sliderInput("fun_sd", "Standard deviation", 32 | min = 0, max = 100, value = 1, step = 0.5, width = "100%") 33 | ), 34 | column(2, 35 | numericInput("sleep_time", "Sleep time (s)", 36 | min = 0, max = 120, value = 90, step = 1) 37 | ) 38 | ), 39 | fluidRow( 40 | column(2, 41 | numericInput("priority", "Priority", 42 | min = 0, max = 10, value = 1, step = 1) 43 | ), 44 | column(3, 45 | textInput("title", "Title", 46 | value = "Task title", width = "100%") 47 | ), 48 | column(3, 49 | textInput("description", "Description", 50 | value = "Task description", width = "100%") 51 | ) 52 | ), 53 | hr(), 54 | fluidRow( 55 | column(6, offset = 3, 56 | actionButton("go_task", "Configure the task", width = "100%") 57 | ) 58 | ) 59 | 60 | ), 61 | tabPanel("View tasks n°1", 62 | fluidRow( 63 | column(12, 64 | tasks_overview_UI("my_id_2") 65 | ) 66 | ), 67 | conditionalPanel(condition = "output.launch_task", 68 | br(), 69 | hr(), 70 | fluidRow( 71 | conditionalPanel(condition = "output.launch_task", 72 | column(12, 73 | div(actionButton("show_task", "Display task result", width = "40%"), 74 | align = "center") 75 | ) 76 | ), 77 | column(12, 78 | conditionalPanel(condition = "input.show_task > 0", 79 | plotOutput("task_plot") 80 | ) 81 | ) 82 | ) 83 | ) 84 | ), 85 | tabPanel("Configure tasks n°2", 86 | fluidRow( 87 | column(2, 88 | numericInput("fun_nb_points_2", "Number of points", 89 | min = 1, max = 10000, value = 100, step = 1) 90 | ), 91 | column(2, 92 | sliderInput("fun_mean_2", "Mean", 93 | min = -10, max = 10, value = 0, step = 0.1, width = "100%") 94 | ), 95 | column(2, 96 | sliderInput("fun_sd_2", "Standard deviation", 97 | min = 0, max = 100, value = 1, step = 0.5, width = "100%") 98 | ), 99 | column(2, 100 | numericInput("sleep_time_2", "Sleep time (s)", 101 | min = 0, max = 120, value = 90, step = 1) 102 | ) 103 | ), 104 | fluidRow( 105 | column(2, 106 | numericInput("priority_2", "Priority", 107 | min = 0, max = 10, value = 1, step = 1) 108 | ), 109 | column(3, 110 | textInput("title_2", "Title", 111 | value = "Task title", width = "100%") 112 | ), 113 | column(3, 114 | textInput("description_2", "Description", 115 | value = "Task description", width = "100%") 116 | ) 117 | ), 118 | hr(), 119 | fluidRow( 120 | column(6, offset = 3, 121 | actionButton("go_task_2", "Configure the task", width = "100%") 122 | ) 123 | ) 124 | 125 | ), 126 | tabPanel("View tasks n°2", 127 | fluidRow( 128 | column(12, 129 | tasks_overview_UI("my_id_2_2") 130 | ) 131 | ), 132 | conditionalPanel(condition = "output.launch_task_2", 133 | br(), 134 | hr(), 135 | fluidRow( 136 | conditionalPanel(condition = "output.launch_task_2", 137 | column(12, 138 | div(actionButton("show_task_2", "Display task result", width = "40%"), 139 | align = "center") 140 | ) 141 | ), 142 | column(12, 143 | conditionalPanel(condition = "input.show_task_2 > 0", 144 | plotOutput("task_plot_2") 145 | ) 146 | ) 147 | ) 148 | ) 149 | ) 150 | ) 151 | ) 152 | ), 153 | tabPanel("Code", 154 | fluidRow( 155 | shinydashboard::tabBox(width = 12, 156 | tabPanel("global.R", 157 | includeMarkdown("www/script/global.md") 158 | ), 159 | tabPanel("server.R", 160 | includeMarkdown("www/script/server.md") 161 | ), 162 | tabPanel("ui.R", 163 | includeMarkdown("www/script/ui.md") 164 | ) 165 | ) 166 | ) 167 | ), 168 | br(), br(), br() 169 | ) 170 | -------------------------------------------------------------------------------- /inst/demo_app_conc/www/css/custom.css: -------------------------------------------------------------------------------- 1 | .selectize-control.multi .selectize-input:after { 2 | content: ''; 3 | display: block; 4 | position: absolute; 5 | top: 50%; 6 | right: 17px; 7 | margin-top: -3px; 8 | width: 0; 9 | height: 0; 10 | border-style: solid; 11 | border-width: 5px 5px 0 5px; 12 | border-color: #333333 transparent transparent transparent; 13 | } 14 | 15 | .ds_app_footer { 16 | position: fixed; 17 | left: 0; 18 | bottom: 0; 19 | width: 100%; 20 | /*background-color: #CC0000; 21 | border-color: #AA0000;*/ 22 | background-color: #430838; 23 | border-color: #0dc6ff; 24 | border-radius: 0; 25 | /*background-image: url('../img/bp_background.png');*/ 26 | color: white; 27 | text-align: center; 28 | z-index: 99999; 29 | } 30 | 31 | 32 | img[alt="img"] { 33 | width: 90%; 34 | display: block; 35 | margin-left: auto; 36 | margin-right: auto 37 | } 38 | 39 | .modal.fade.in { 40 | z-index: 100000 !important; 41 | } 42 | 43 | .navbar-default { 44 | /*background-color: #CC0000; 45 | border-color: #AA0000; 46 | border-radius: 0; 47 | background-image: url('../img/bp_background.png');*/ 48 | background-color: #430838; 49 | z-index: 99999; 50 | display: inline-block; 51 | } 52 | 53 | 54 | .navbar-default .navbar-brand, 55 | .navbar-default .navbar-brand:hover, 56 | .navbar-default .navbar-brand:focus { 57 | display: inline-block; 58 | color: #FFF; 59 | padding-top:20px !important; 60 | padding-bottom:20px !important; 61 | font-size: 1.6em !important; 62 | } 63 | 64 | .navbar-default .navbar-nav > li > a { 65 | display: inline-block; 66 | color:#FFF; 67 | opacity:.5; 68 | font-weight:500; 69 | padding-top:20px !important; 70 | padding-bottom:20px !important; 71 | font-size: 1.35em !important; 72 | } 73 | 74 | .navbar-default .navbar-nav > li > a:hover { 75 | display: inline-block; 76 | text-decoration:none; 77 | color:#FFF; 78 | opacity:1 79 | } 80 | 81 | .navbar-default .navbar-nav > li > a:focus { 82 | background-color: #430838; 83 | border-bottom:5px solid #e6e6e6; 84 | display: inline-block; 85 | color:#FFF; 86 | opacity:1 87 | } 88 | 89 | .navbar-default .navbar-nav > .active > a, 90 | .navbar-default .navbar-nav > .active > a:hover, 91 | .navbar-default .navbar-nav > .active > a:focus { 92 | background-color: #430838; 93 | border-bottom:5px solid #e6e6e6; 94 | color:#FFF; 95 | opacity:1 96 | } 97 | 98 | .navbar-default .navbar-text { 99 | color: #FFF; 100 | } 101 | 102 | .navbar-default .navbar-toggle { 103 | border-color: #AA0000; 104 | } 105 | 106 | .navbar-default .navbar-toggle:hover, 107 | .navbar-default .navbar-toggle:focus { 108 | background-color: #AA0000; 109 | } 110 | 111 | .navbar-default .navbar-toggle .icon-bar { 112 | background-color: #FFF; 113 | } 114 | 115 | 116 | .ribbon { 117 | position: fixed; 118 | left: 0; 119 | top: 0; 120 | z-index: 100000; 121 | margin-left: 0cm; 122 | margin-top: 0.1cm; 123 | display: block; 124 | height: 65px; 125 | text-decoration: none; 126 | overflow-x: hidden; 127 | } 128 | 129 | body,p,html{ 130 | font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif; 131 | } 132 | 133 | h1,h2,h3,h4,h5,h6{ 134 | font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif; 135 | color: #430838; 136 | /*font-weight: bold*/ 137 | } 138 | 139 | a{ 140 | color: black; 141 | } 142 | a:hover { 143 | color: #e6e6e6; 144 | } 145 | -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/conf_file.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/conf_file.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/custom_cron.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/custom_cron.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/default_cron.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/default_cron.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/demo_app_conf_task.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/demo_app_conf_task.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/demo_app_view_task_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/demo_app_view_task_1.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/demo_app_view_task_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/demo_app_view_task_2.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/launch_task_shiny.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/launch_task_shiny.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/launch_task_shiny_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/launch_task_shiny_2.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/launch_task_shiny_code.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/launch_task_shiny_code.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/see_tasks_shiny.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/see_tasks_shiny.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/see_tasks_shiny_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/see_tasks_shiny_2.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/figures/see_tasks_shiny_code.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/figures/see_tasks_shiny_code.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/img/github.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/img/github.png -------------------------------------------------------------------------------- /inst/demo_app_conc/www/img/img-datastorm-logo-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/img/img-datastorm-logo-white.png -------------------------------------------------------------------------------- /inst/demo_app_conc/www/img/logoDS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/img/logoDS.png -------------------------------------------------------------------------------- /inst/demo_app_conc/www/img/rpivot.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastorm-open/shinybatch/f6cefc9c58683406f674b1324a7f27c3cd67941e/inst/demo_app_conc/www/img/rpivot.PNG -------------------------------------------------------------------------------- /inst/demo_app_conc/www/script/global.md: -------------------------------------------------------------------------------- 1 | ``` r 2 | require(shiny) 3 | require(shinydashboard) 4 | require(DT) 5 | require(shinybatch) 6 | 7 | # create directory for conf 8 | dir_conf <- paste0(tempdir(), "/conf") 9 | if (dir.exists(dir_conf)) unlink(dir_conf, recursive = TRUE) 10 | dir.create(dir_conf, recursive = T) 11 | 12 | # get fun path 13 | fun_path = system.file("ex_fun/sb_fun_ex_demo_app.R", package = "shinybatch") 14 | 15 | # check if cron already existed 16 | # else create it 17 | exists_cron <- scheduler_exist("cr_sc_demo") 18 | if(!exists_cron){ 19 | 20 | # init dir_scheduler 21 | dir_scheduler <- paste0(tempdir(), "/cron") 22 | if (dir.exists(dir_scheduler)) unlink(dir_scheduler, recursive = TRUE) 23 | dir.create(dir_scheduler, recursive = T) 24 | 25 | scheduler_add(dir_scheduler = dir_scheduler, 26 | dir_conf = dir_conf, 27 | max_runs = 1, 28 | head_rows = NULL, 29 | taskname = "cr_sc_demo") 30 | } 31 | ``` 32 | -------------------------------------------------------------------------------- /inst/demo_app_conc/www/script/intro.md: -------------------------------------------------------------------------------- 1 | # shinybatch 2 | 3 | This package provides a simple framework to create, launch *automatically* and retrieve *time-consuming operations* (tasks) **in batch mode** from a **Shiny app**. 4 | 5 | The tasks are automatically launched using a scheduler, e.g. a timer that periodically launches a (batch) operation. 6 | 7 | - with a CRON in Linux/Mac, using package **cronR** 8 | - with Windows Task Scheduler in Windows, using package **taskscheduleR** 9 | - or defined directly with OS tools 10 | 11 | ### Installation 12 | 13 | ``` r 14 | devtools::install_github("datastorm-open/shinybatch") 15 | ``` 16 | 17 | ### Main functions 18 | 19 | - **configure_task** : create a *.yml* file filed with operation params (fun path, fun args, priority, ...), 20 | - **run_task** : run a selected task from a *.yml* file. Used by **launcher**, 21 | - **launcher** : select and run the task(s) with highest priority, calling **run_task** in batch mode with *RScript* command, 22 | - **scheduler_init** : (opt: create the cron R script) create the file to be launched by the *scheduler*, 23 | - **scheduler_add** : (opt: create the cron file, and by default the cron R script) and start the *scheduler* which will read the file at the given frequency to launch batch operations, 24 | - **configure_task_server** : define a task and call *configure_task()* in the Shiny app, 25 | - **tasks_overview_server** : display tasks & retrieve results within the Shiny app. 26 | 27 | 28 | ### Definition of a task 29 | 30 | *configure_task()* 31 | 32 | A task is defined by its *.yml* file that contains the following informations : 33 | 34 | ``` yml 35 | run_info: 36 | date_creation: 2020-04-24 15:21:00 37 | date_start: N/A 38 | date_end: N/A 39 | priority: 1.0 40 | status: waiting 41 | descriptive: 42 | title: my_title 43 | description: my_descr 44 | function: 45 | path: /path/to/my_fun 46 | name: my_fun_name 47 | args: 48 | x: 1.0 49 | 'y': 50 | _path: /path/to/task/dir/inputs/y.RDS 51 | z: 52 | _path: /path/to/task/dir/inputs/z.RDS 53 | dir: /path/to/task/dir/ 54 | ``` 55 | 56 | 57 | The ``run_info`` part contains general informations about the task. 58 | 59 | Priority can be any number, with 0 as default. The highest the priority, the sooner it is launched. 60 | 61 | Valid status are: 62 | 63 | - **waiting** 64 | - **running** 65 | - **finished** 66 | - **error** 67 | 68 | The ``descriptive`` part contains free informations given by the user. The title and description fields are only example. 69 | 70 | The ``function`` part contains the location of the fun **R** script (for sourcing) and its name (for calling). The script must have all necessary ressources (packages, variables, functions) for execute the main function : 71 | 72 | ``` r 73 | # Load package(s) (if needed) 74 | require(data.table) 75 | 76 | # source script(s) (if needed) 77 | source("/path/to/script") 78 | 79 | # Load data (if needed) 80 | data <- readRDS("/path/to/script") 81 | 82 | # Define main function (needed !) 83 | my_fun_name <- function(x, y, z){ 84 | ... 85 | } 86 | ``` 87 | 88 | The ``args`` part contains either the argument itself if it is of length 1 or the location of the argument (in *dir_conf/inputs/arg_name.RDS*), complex arguments are storing as *.RDS* 89 | 90 | The ``dir`` argument contains the location of the directory in which is stored the *conf.yml* file. 91 | 92 | When a task has been succesfully run, some fields are updated: 93 | 94 | - ``date_start`` and ``date_end`` are filled, 95 | - ``status`` is set to 'running', then to 'finished'. 96 | 97 | Some outputs are created: 98 | 99 | - if wanted, the result of the task, in other words the output of the main function (in *dir_conf/output/res.RDS*) 100 | - the log of the run (in *dir_conf/output/log_run.txt*) 101 | 102 | ### Description of the launcher 103 | 104 | *launcher()* 105 | 106 | The launcher retrieves all the tasks in a main directory and build a the table of their *run_info*. Based on this, it verifies that there are tasks with *status* that allow a run, e.g. all but those in *ignore_status*. Then, if the maximum number of simultaneously running tasks is not reached, it launches new tasks according to their priority, in batch mode with *RScript* command (and calling *run_task()*) 107 | 108 | 109 | The task with higher priority is defined as the one: 110 | 111 | - with *status* not in *ignore_status* (default is all but **waiting**), 112 | - which has the highest *priority*, 113 | - and then the oldest *date_creation*. 114 | 115 | ### Description of the scheduler 116 | 117 | Before calling the scheduler, we first need to create the file that it will launch. We can use **scheduler_init()**. By default, it looks like this (*scheduler_script.R*) : 118 | 119 | ```` 120 | #!/usr/bin/env Rscript 121 | args = commandArgs(trailingOnly = TRUE) 122 | 123 | shinybatch::launcher(dir_path = '/path/to/main_directory/', 124 | max_runs = 1, 125 | ignore_status = c('running','finished','error'), 126 | delay_reruns = TRUE 127 | ) 128 | ```` 129 | 130 | ... but the head lines can be customized by filling the *head_rows* argument. 131 | 132 | 133 | Once the file has been created, the cron has to be defined. You can use directly ``cron`` on linux or the ``Task Scheduler`` on windows, or the **scheduler_add** function. The default command is : 134 | 135 | ``Rscript /path/to/scheduler_script.R`` 136 | 137 | *N.B :* **scheduler_add** creates by default the **R** scheduler script using **scheduler_init** 138 | 139 | ### Example 140 | 141 | **sb_fun_ex.R** 142 | 143 | ``` r 144 | sb_fun_ex <- function(x, y, z) { 145 | res <- x + y 146 | message('Running !') 147 | warning("Complex (or not) variable z is not used...!", call. = FALSE) 148 | res 149 | } 150 | ``` 151 | 152 | **Configure a task** 153 | ``` r 154 | require(shinybatch) 155 | 156 | ?configure_task 157 | 158 | # create temporary directory 159 | dir <- tempdir() 160 | 161 | # create and save conf 162 | conf <- configure_task( 163 | dir_path = dir, 164 | conf_descr = list( 165 | title = "my_title", 166 | description = "my_descr" 167 | ), 168 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch"), # as an example, 169 | fun_name = "sb_fun_ex", 170 | fun_args = list( 171 | x = 1, 172 | y = 0:4, 173 | z = iris 174 | ), 175 | priority = 1) 176 | 177 | # check results 178 | list.files(conf$dir, recursive = T) 179 | # [1] "conf.yml" "inputs/y.RDS" "inputs/z.RDS" 180 | 181 | read_conf <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 182 | 183 | read_conf$run_info 184 | # $date_creation 185 | # [1] "2021-03-02 16:54:24" 186 | # 187 | # $date_start 188 | # [1] "N/A" 189 | # 190 | # $date_end 191 | # [1] "N/A" 192 | # 193 | # $priority 194 | # [1] 1 195 | # 196 | # $status 197 | # [1] "waiting" 198 | 199 | read_conf$args$x 200 | # [1] 1 201 | 202 | y <- readRDS(paste0(conf$dir, "inputs/y.RDS")) 203 | # [1] 0 1 2 3 4 204 | 205 | z <- readRDS(paste0(conf$dir, "inputs/z.RDS")) 206 | ``` 207 | 208 | **Run one given task (for demo/test)** 209 | 210 | ``` r 211 | ?run_task 212 | 213 | run_task(paste0(conf$dir, "conf.yml")) 214 | 215 | # catch results 216 | list.files(conf$dir, recursive = T) 217 | # output directory with log & result 218 | # [1] "conf.yml" "inputs/y.RDS" 219 | # [3] "inputs/z.RDS" "output/log_run_20210302_1656_1080.txt" 220 | # [5] "output/res.RDS" 221 | 222 | conf_update <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 223 | conf_update$run_info 224 | # $date_creation 225 | # [1] "2021-03-02 16:54:24" 226 | # 227 | # $date_start 228 | # [1] "2021-03-02 16:56:10" 229 | # 230 | # $date_end 231 | # [1] "2021-03-02 16:56:10" 232 | # 233 | # $priority 234 | # [1] 1 235 | # 236 | # $status 237 | # [1] "finished" 238 | 239 | 240 | output <- readRDS(paste0(conf$dir, "output/res.RDS")) 241 | #[1] 1 2 3 4 5 242 | 243 | log <- read.delim(list.files(paste0(conf$dir, "output/"), pattern = "log_run", full.names = T), header = F) 244 | # [2021-03-02 16:56:10] [INFO] Starting task execution... 245 | # [2021-03-02 16:56:10] [INFO] Running ! 246 | # [2021-03-02 16:56:10] [WARN] Complex (or not) variable z is not used...! 247 | # [2021-03-02 16:56:10] [INFO] ... task terminated. 248 | ``` 249 | 250 | **Use scheduler to launch the tasks** 251 | 252 | ``` r 253 | ?scheduler_add 254 | 255 | # create temporary directory for conf 256 | dir_conf <- paste0(tempdir(), "/conf/") 257 | dir.create(dir_conf, recursive = T) 258 | 259 | # create 2 confs 260 | conf_1 <- configure_task( 261 | dir_path = dir_conf, 262 | conf_descr = list( 263 | title_1 = "my_title_1", 264 | description_1 = "my_descr_1" 265 | ), 266 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch"), # as an example, 267 | fun_name = "sb_fun_ex", 268 | fun_args = list( 269 | x = 0, 270 | y = 0:4, 271 | z = iris 272 | ), 273 | priority = 1) 274 | 275 | conf_2 <- configure_task( 276 | dir_path = dir_conf, 277 | conf_descr = list( 278 | title_1 = "my_title_2", 279 | description_1 = "my_descr_2" 280 | ), 281 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch"), # as an example, 282 | fun_name = "sb_fun_ex", 283 | fun_args = list( 284 | x = 0, 285 | y = 0:4, 286 | z = iris 287 | ), 288 | priority = 2) 289 | 290 | # on LINUX -> Needs cronR package 291 | # on Windows -> Needs taskscheduleR package 292 | 293 | scheduler_add( 294 | dir_scheduler = tempdir(), 295 | dir_conf = dir_conf, 296 | max_runs = 1, 297 | create_file = T, 298 | head_rows = NULL, 299 | taskname = "cron_script_ex" 300 | ) 301 | 302 | scheduler_ls() # display existing crons 303 | 304 | # wait up to 1 min for conf_2 and up to 2 mins for conf_1 305 | yaml::read_yaml(paste0(conf_1$dir, "/conf.yml"))$run_info$status 306 | yaml::read_yaml(paste0(conf_2$dir, "/conf.yml"))$run_info$status 307 | 308 | scheduler_rm(id = "cron_script_ex") # kill selected cron 309 | ``` 310 | 311 | **Shiny modules** 312 | 313 | These modules contain the basic framework to use all the previous functions in a Shiny app. 314 | Both are used in the demo app which presents a simple usecase. 315 | 316 | - **Configure a new task** 317 | 318 | ![img](figures/launch_task_shiny.PNG) 319 | 320 | ![img](figures/launch_task_shiny_2.PNG) 321 | 322 | call: 323 | 324 | ``` r 325 | ?configure_task_server 326 | 327 | # ui : just create an actionButton 328 | actionButton("go_task", "Configure the task !") 329 | 330 | # server 331 | # call module to configure a task 332 | # connect app inputs to the module 333 | callModule(configure_task_server, "my_id_1", 334 | btn = reactive(input$go_task), 335 | dir_path = dir_conf, 336 | conf_descr = reactive( 337 | list( 338 | "title" = input$title, 339 | "description" = input$description 340 | ) 341 | ), 342 | fun_path = paste0(dir_fun, "/fun_script.R"), 343 | fun_name = "my_fun", 344 | fun_args = reactive( 345 | list( 346 | n = input$fun_nb_points, 347 | mean = input$fun_mean, 348 | sd = input$fun_sd, 349 | sleep = input$sleep_time 350 | ) 351 | ), 352 | priority = reactive(input$priority) 353 | ) 354 | ``` 355 | 356 | 357 | - **Display configured tasks** 358 | 359 | ![img](figures/see_tasks_shiny.PNG) 360 | 361 | ![img](figures/see_tasks_shiny_2.PNG) 362 | 363 | 364 | call: 365 | 366 | ``` r 367 | ?tasks_overview_UI 368 | 369 | # ui 370 | tasks_overview_UI("my_id_2") 371 | 372 | # server 373 | # call module to view tasks 374 | sel_task <- callModule( 375 | tasks_overview_server, "my_id_2", 376 | dir_path = dir_conf, 377 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 378 | allowed_run_info_cols = NULL, 379 | allowed_function_cols = NULL, 380 | allow_descr = T, 381 | allow_args = T 382 | ) 383 | ``` 384 | 385 | This module returns : 386 | 387 | - the status of the selected line (one run) of the summary table, 388 | - the path to the directory in which its output is stored. 389 | 390 | Thus we know when a run is finished and we can load its result to reuse/display it : (readRDS(paste0(path, "/res.RDS"))). 391 | 392 | **Demo app** 393 | 394 | A demo app to create and automatically launch an example task : the generation of normally distributed observations. 395 | 396 | - **global** : the path to the confs directory, the path to the script of the function to be run, the call to scheduler_add() ; 397 | - **ui** : shiny inputs (description args for the conf ; parameters for the function to be called by the cron) ; 398 | - **server** : a renderPlot (a graph of the data create in a run). 399 | 400 | As a credible usecase, the results of the runs are retrieved and can be displayed. 401 | 402 | ``` r 403 | runApp(system.file("demo_app", package = "shinybatch")) 404 | ``` 405 | 406 | ![img](figures/demo_app_conf_task.PNG) 407 | 408 | ![img](figures/demo_app_view_task_1.PNG) 409 | 410 | ![img](figures/demo_app_view_task_2.PNG) 411 | 412 |
413 | -------------------------------------------------------------------------------- /inst/demo_app_conc/www/script/server.md: -------------------------------------------------------------------------------- 1 | ``` r 2 | # call module to configure a task 3 | # connect app inputs to the module 4 | callModule(configure_task_server, "my_id_1", 5 | btn = reactive(input$go_task), 6 | dir_path = dir_conf, 7 | conf_descr = reactive(list("title" = input$title, 8 | "description" = input$description)), 9 | fun_path = fun_path, 10 | fun_name = "my_fun", 11 | fun_args = reactive(list(n = input$fun_nb_points, 12 | mean = input$fun_mean, 13 | sd = input$fun_sd, 14 | sleep = input$sleep_time)), 15 | priority = reactive(input$priority)) 16 | 17 | # call module to view tasks 18 | sel_task <- callModule(tasks_overview_server, "my_id_2", 19 | dir_path = dir_conf, 20 | update_mode = "reactive", 21 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 22 | allowed_run_info_cols = NULL, 23 | allowed_function_cols = NULL, 24 | allow_descr = T, 25 | allow_args = T) 26 | 27 | # check if sel_task can be displayed 28 | output$launch_task <- reactive({ 29 | ! is.null(sel_task()) && length(sel_task()) > 0 && sel_task()$status == "finished" 30 | }) 31 | outputOptions(output, "launch_task", suspendWhenHidden = FALSE) 32 | 33 | # display task 34 | output$task_plot <- renderPlot({ 35 | cpt <- input$show_task 36 | sel_task <- sel_task() 37 | 38 | isolate({ 39 | if (cpt > 0 && length(sel_task) > 0 && sel_task$status == "finished") { 40 | 41 | data <- readRDS(paste0(sel_task$path, "/res.RDS")) 42 | 43 | plot(density(data), col = "#3c8dbc", lwd = 2, 44 | main = "Density plot of generated observations", xlab = "", 45 | ylab = "Density", axes = F) 46 | axis(1) 47 | axis(2) 48 | } 49 | }) 50 | }) 51 | ``` 52 | -------------------------------------------------------------------------------- /inst/demo_app_conc/www/script/ui.md: -------------------------------------------------------------------------------- 1 | ``` r 2 | fluidRow( 3 | tabBox(width = 12, 4 | tabPanel("Configure tasks", 5 | fluidRow( 6 | column(2, 7 | numericInput("fun_nb_points", "Number of points", 8 | min = 1, max = 10000, value = 100, step = 1) 9 | ), 10 | column(2, 11 | sliderInput("fun_mean", "Mean", 12 | min = -10, max = 10, value = 0, step = 0.1, width = "100%") 13 | ), 14 | column(2, 15 | sliderInput("fun_sd", "Standard deviation", 16 | min = 0, max = 100, value = 1, step = 0.5, width = "100%") 17 | ), 18 | column(2, 19 | numericInput("sleep_time", "Sleep time (s)", 20 | min = 0, max = 30, value = 0, step = 1) 21 | ) 22 | ), 23 | fluidRow( 24 | column(2, 25 | numericInput("priority", "Priority", 26 | min = 0, max = 10, value = 1, step = 1) 27 | ), 28 | column(3, 29 | textInput("title", "Title", 30 | value = "Task title", width = "100%") 31 | ), 32 | column(3, 33 | textInput("description", "Description", 34 | value = "Task description", width = "100%") 35 | ) 36 | ), 37 | hr(), 38 | fluidRow( 39 | column(6, offset = 3, 40 | actionButton("go_task", "Configure the task", width = "100%") 41 | ) 42 | ) 43 | 44 | ), 45 | tabPanel("View tasks", 46 | fluidRow( 47 | column(12, 48 | tasks_overview_UI("my_id_2") 49 | ) 50 | ), 51 | conditionalPanel(condition = "output.launch_task", 52 | br(), 53 | hr(), 54 | fluidRow( 55 | conditionalPanel(condition = "output.launch_task", 56 | column(12, 57 | div(actionButton("show_task", "Display task result", width = "40%"), 58 | align = "center") 59 | ) 60 | ), 61 | column(12, 62 | conditionalPanel(condition = "input.show_task > 0", 63 | plotOutput("task_plot") 64 | ) 65 | ) 66 | ) 67 | ) 68 | ) 69 | ) 70 | ) 71 | ``` 72 | -------------------------------------------------------------------------------- /inst/ex_fun/sb_fun_demo_app_python.R: -------------------------------------------------------------------------------- 1 | wrapper_py <- function(vect) { 2 | require(reticulate) 3 | 4 | env <- py_run_file(system.file("/ex_fun/sb_fun_demo_app_python.py", package = "shinybatch")) ; 5 | env$apply_regexprs(vect = vect, 6 | regexprs = reticulate::dict(list("input" = "output"))) 7 | } -------------------------------------------------------------------------------- /inst/ex_fun/sb_fun_demo_app_python.py: -------------------------------------------------------------------------------- 1 | ################## 2 | # py script test # 3 | ################## 4 | 5 | import re 6 | 7 | 8 | # fun to be applied 9 | def apply_regexprs(vect, 10 | regexprs): 11 | 12 | # controls 13 | vect_type = type(vect).__name__ 14 | if vect is None or not vect_type in ["Series", "str", "list"]: 15 | raise Exception("vect must be of class , or .") 16 | else: 17 | if vect_type == 'Series': 18 | vect = list(vect) 19 | if regexprs is None or not type(regexprs).__name__ in ["dict", "list"]: 20 | raise Exception("regexprs must be of class or ") 21 | 22 | # compile regexpr to save time (2-3 times faster) 23 | compiled_reg = [re.compile(x) for x in regexprs] 24 | 25 | # handle case with multiple elements in vect 26 | def map_fun(vect_): 27 | for reg, val in zip(compiled_reg, regexprs.values()): 28 | if vect_ is not None: 29 | vect_ = reg.sub(val, vect_) 30 | return vect_ 31 | 32 | # handle case when only one element in regexprs and vect 33 | if type(vect).__name__ == 'str': 34 | vect = [vect] 35 | 36 | # apply all regexprs on each element of vect 37 | vect = list(map(map_fun, vect)) 38 | 39 | if vect_type == 'Series': 40 | vect = pd.Series(vect) 41 | 42 | return vect 43 | -------------------------------------------------------------------------------- /inst/ex_fun/sb_fun_ex.R: -------------------------------------------------------------------------------- 1 | sb_fun_ex <- function(x, y, z) { 2 | res <- x + y 3 | message('Running !') 4 | warning("Complex (or not) variable z is not used...!") 5 | res 6 | } 7 | -------------------------------------------------------------------------------- /inst/ex_fun/sb_fun_ex_demo_app.R: -------------------------------------------------------------------------------- 1 | my_fun <- function(n, mean, sd, sleep) { 2 | Sys.sleep(sleep) 3 | rnorm(n, mean, sd) 4 | } -------------------------------------------------------------------------------- /inst/scripts/test.R: -------------------------------------------------------------------------------- 1 | 2 | dir_conf <- paste0(getwd(), "/conf") 3 | unlink(dir_conf, recursive = T) 4 | dir.create(dir_conf, recursive = T) 5 | 6 | # get fun 7 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 8 | fun_name = "sb_fun_ex" 9 | 10 | # create and save conf 11 | conf <- configure_task(dir_path = dir_conf, 12 | conf_descr = list(title = "my_title", 13 | description = "my_descr"), 14 | fun_path = fun_path, 15 | fun_name = fun_name, 16 | fun_args = list(x = 1, 17 | y = 0:4, 18 | z = iris), 19 | priority = 1) 20 | 21 | # conf_init <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 22 | # y <- readRDS(paste0(conf$dir, "inputs/y.RDS")) 23 | # z <- readRDS(paste0(conf$dir, "inputs/z.RDS")) 24 | # 25 | # conf_path = paste0(conf$dir, "conf.yml") 26 | # ignore_status = c("running", "finished", "error") 27 | # save_rds = TRUE 28 | # return = TRUE 29 | 30 | # run_task(paste0(conf$dir, "conf.yml"), save_rds = FALSE, ignore_status = NULL) 31 | 32 | launcher(dir_path = dir_conf) 33 | launcher(dir_path = dir_conf, ignore_status = "error") 34 | # catch results 35 | list.files(conf$dir) 36 | conf_update <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 37 | output <- readRDS(paste0(conf$dir, "output/res.RDS")) 38 | log <- read.delim(list.files(paste0(conf$dir, "output/"), 39 | pattern = "log_run", full.names = T), header = F) -------------------------------------------------------------------------------- /man/configuration_info.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/conf_to_dt.R 3 | \name{conf_to_dt} 4 | \alias{conf_to_dt} 5 | \alias{dir_conf_to_dt} 6 | \title{Convert a list of task configurations into two data.tables of global and individual features.} 7 | \usage{ 8 | conf_to_dt( 9 | confs, 10 | allowed_run_info_cols = c("date_creation", "date_start", "date_end", "priority", 11 | "status"), 12 | allow_descr = TRUE, 13 | allowed_function_cols = c("path", "name"), 14 | allow_args = TRUE 15 | ) 16 | 17 | dir_conf_to_dt( 18 | dir_path, 19 | allowed_run_info_cols = c("date_creation", "date_start", "date_end", "priority", 20 | "status"), 21 | allow_descr = TRUE, 22 | allowed_function_cols = c("path", "name"), 23 | allow_args = TRUE 24 | ) 25 | } 26 | \arguments{ 27 | \item{confs}{\code{list of list}. List of conf list(s) from yaml file(s).} 28 | 29 | \item{allowed_run_info_cols}{\code{characteror or boolean} (c("date_creation", "date_start", "date_end", "priority", "status")). Run info elements to be kept.} 30 | 31 | \item{allow_descr}{\code{boolean or character} (TRUE). Either a boolean specifying whether or not to keep descr elements, or column names.} 32 | 33 | \item{allowed_function_cols}{\code{character or boolean} (c("names", "path")). Function elements to be kept.} 34 | 35 | \item{allow_args}{\code{boolean or character} (TRUE). Either a boolean specifying whether or not to keep args elements, or column names.} 36 | 37 | \item{dir_path}{\code{character}. Path to the directory with tasks.} 38 | } 39 | \value{ 40 | a list of two tables 'tbl_global' and 'tbl_idv': 41 | \itemize{ 42 | \item{tbl_global} {contains the global features of the configurations (.$run_info and .$descriptive)} 43 | \item{tbl_idv} {contains the individual features of the configurations (.$function and .$args)} 44 | } 45 | } 46 | \description{ 47 | Convert a list of task configurations into two data.tables of global and individual features. 48 | } 49 | \examples{ 50 | 51 | \donttest{ 52 | 53 | dir_conf <- paste0(tempdir(), "/conf") 54 | dir.create(dir_conf, recursive = TRUE) 55 | 56 | # ex fun 57 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 58 | fun_name = "sb_fun_ex" 59 | 60 | # create 2 confs 61 | conf_1 <- configure_task(dir_path = dir_conf, 62 | conf_descr = list(title_1 = "my_title_1", 63 | description_1 = "my_descr_1"), 64 | fun_path = fun_path, 65 | fun_name = fun_name, 66 | fun_args = list(x = 0, 67 | y = 0:4, 68 | z = iris), 69 | priority = 1) 70 | conf_2 <- configure_task(dir_path = dir_conf, 71 | conf_descr = list(title_2 = "my_title_2", 72 | description_2 = "my_descr_2"), 73 | fun_path = fun_path, 74 | fun_name = fun_name, 75 | fun_args = list(x = 1, 76 | y = 0:4, 77 | z = iris), 78 | priority = 2) 79 | 80 | # retrieve information about all tasks in main directory 81 | dir_conf_to_dt(dir_conf, allowed_run_info_cols = FALSE) 82 | 83 | dir_conf_to_dt(dir_conf, 84 | allow_descr = FALSE, 85 | allow_args = FALSE) 86 | 87 | dir_conf_to_dt(dir_conf, 88 | allowed_run_info_cols = c("status", "date_creation"), 89 | allowed_function_cols = c("path")) 90 | 91 | dir_conf_to_dt(dir_conf, 92 | allowed_run_info_cols = NULL, 93 | allowed_function_cols = NULL) 94 | 95 | dir_conf_to_dt(dir_conf, 96 | allowed_run_info_cols = "", 97 | allowed_function_cols = "", 98 | allow_descr = FALSE, 99 | allow_args = FALSE) 100 | 101 | # or just on some tasks ? 102 | info_conf_1 <- conf_1 103 | info_conf_2 <- yaml::read_yaml(file.path(conf_2$dir, "conf.yml")) 104 | 105 | conf_to_dt(list(info_conf_1, info_conf_2)) 106 | 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /man/configure_task.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/configure_task.R 3 | \name{configure_task} 4 | \alias{configure_task} 5 | \title{Initialize a configuration file for a future task} 6 | \usage{ 7 | configure_task( 8 | dir_path, 9 | fun_path, 10 | fun_name, 11 | conf_descr = NULL, 12 | fun_args = NULL, 13 | priority = 0L, 14 | compress = TRUE, 15 | call. = TRUE 16 | ) 17 | } 18 | \arguments{ 19 | \item{dir_path}{\code{character}. Tasks location (parent directory).} 20 | 21 | \item{fun_path}{\code{character}. Path to the script of the function.} 22 | 23 | \item{fun_name}{\code{character}. Name of the function in fun_path script.} 24 | 25 | \item{conf_descr}{\code{named list} (NULL). Optional description fields.} 26 | 27 | \item{fun_args}{\code{named list} (NULL). Args of the function, must all be named.} 28 | 29 | \item{priority}{\code{numeric} (0L). Number used to define which task should be launched first using \code{\link[shinybatch]{launcher}}} 30 | 31 | \item{compress}{\code{logical or character} (TRUE). Either a logical specifying whether or not to use "gzip" compression, or one of "gzip", "bzip2" or "xz" 32 | to indicate the type of compression to be used for function result} 33 | 34 | \item{call.}{\code{logical} (TRUE) logical, indicating if the call should become part of the error message (in log file)} 35 | } 36 | \value{ 37 | a list containing the conf fields. Attribute 'path' of the result contains the path to the conf directory. 38 | The arg field contains either the location of the argument (in "dir_path/inputs/arg_name.RDS") or 39 | the argument itself if it is of length 1. 40 | } 41 | \description{ 42 | Initialize a configuration file for a future task 43 | } 44 | \examples{ 45 | 46 | \donttest{ 47 | 48 | 49 | # create temporary directory 50 | dir <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 51 | dir.create(dir, recursive = TRUE) 52 | 53 | # create and save conf 54 | conf <- configure_task(dir_path = dir, 55 | conf_descr = list(title = "my_title", 56 | description = "my_descr"), 57 | fun_path = dir, # as an example 58 | fun_name = "my_fun_name", 59 | fun_args = list(x = 1, 60 | y = 0:4, 61 | z = iris), 62 | priority = 1) 63 | 64 | # catch results 65 | list.files(conf$dir) 66 | read_conf <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 67 | y <- readRDS(paste0(conf$dir, "inputs/y.RDS")) 68 | z <- readRDS(paste0(conf$dir, "inputs/z.RDS")) 69 | 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /man/input_btns.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{input_btns} 4 | \alias{input_btns} 5 | \title{create button for DT table} 6 | \usage{ 7 | input_btns( 8 | inputId, 9 | col_value, 10 | tooltip, 11 | label = "", 12 | icon = NULL, 13 | status = "primary" 14 | ) 15 | } 16 | \arguments{ 17 | \item{inputId}{\code{character}.\code{\link{actionButton}}.} 18 | 19 | \item{col_value}{\code{vector}. \code{\link{actionButton}}.} 20 | 21 | \item{tooltip}{\code{character}. \code{\link{actionButton}}.} 22 | 23 | \item{label}{\code{character}. \code{\link{actionButton}}.} 24 | 25 | \item{icon}{\code{character}. \code{\link{actionButton}}.} 26 | 27 | \item{status}{\code{character}. \code{\link{actionButton}}.} 28 | } 29 | \value{ 30 | a shiny btn. 31 | } 32 | \description{ 33 | create button for DT table 34 | } 35 | -------------------------------------------------------------------------------- /man/launcher.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/launcher.R 3 | \name{launcher} 4 | \alias{launcher} 5 | \title{Evaluate and trigger the number of tasks to launch to reach the maximum allowed} 6 | \usage{ 7 | launcher( 8 | dir_path, 9 | max_runs = 1, 10 | ignore_status = c("running", "finished", "timeout", "error"), 11 | delay_reruns = TRUE, 12 | compress = TRUE, 13 | verbose = FALSE, 14 | timeout = Inf 15 | ) 16 | } 17 | \arguments{ 18 | \item{dir_path}{\code{character}. Where to find the tasks directory (one or more). If several, log file are present in first directory} 19 | 20 | \item{max_runs}{\code{integer}. Maximum number of simultaneous running tasks.} 21 | 22 | \item{ignore_status}{\code{character} (c("running", "finished", "timeout", "error")). Status to be ignored when launching tasks.} 23 | 24 | \item{delay_reruns}{\code{boolean} (TRUE). When "running", "finished", "timeout" or "error" are not in ignore_status, use the date of the last run instead of 25 | the date of creation of the task to compute the order of (re)run for these tasks. The priority still applies.} 26 | 27 | \item{compress}{\code{logical or character} (TRUE). Either a logical specifying whether or not to use "gzip" compression, or one of "gzip", "bzip2" or "xz" to indicate the type of compression to be used.} 28 | 29 | \item{verbose}{\code{logical} See running task message ? Default to FALSE} 30 | 31 | \item{timeout}{\code{numeric} Minute. Long task running more than \code{timeout} (perhaps killed with server restart or full memory) are set to "timeout" to enable running other waiting tasks} 32 | } 33 | \value{ 34 | the number of launched tasks. 35 | } 36 | \description{ 37 | Evaluate and trigger the number of tasks to launch to reach the maximum allowed 38 | } 39 | \examples{ 40 | 41 | \donttest{ 42 | 43 | # create temporary directory for conf 44 | dir_conf <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 45 | dir.create(dir_conf, recursive = TRUE) 46 | 47 | # ex fun 48 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 49 | fun_name = "sb_fun_ex" 50 | 51 | # create 2 confs 52 | conf_1 <- configure_task(dir_path = dir_conf, 53 | conf_descr = list( 54 | title_1 = "my_title_1", 55 | description_1 = "my_descr_1" 56 | ), 57 | fun_path = fun_path, 58 | fun_name = fun_name, 59 | fun_args = list( 60 | x = 0, 61 | y = 0:4, 62 | z = iris 63 | ), 64 | priority = 1) 65 | 66 | conf_2 <- configure_task(dir_path = dir_conf, 67 | conf_descr = list( 68 | title_2 = "my_title_2", 69 | description_2 = "my_descr_2" 70 | ), 71 | fun_path = fun_path, 72 | fun_name = fun_name, 73 | fun_args = list( 74 | x = 1, 75 | y = 0:4, 76 | z = iris 77 | ), 78 | priority = 2) 79 | 80 | launcher(dir_conf, verbose = TRUE) 81 | # display res of conf_2 in /output dir 82 | Sys.sleep(2) # waiting scheduler computation 83 | readRDS(paste0(conf_2$dir, "output/res.RDS")) 84 | 85 | launcher(dir_conf, verbose = TRUE) 86 | # display res of conf_1 in /output dir 87 | Sys.sleep(2) # waiting scheduler computation 88 | readRDS(paste0(conf_1$dir, "output/res.RDS")) 89 | 90 | launcher(dir_conf, verbose = TRUE) 91 | 92 | # launch again a finished task 93 | launcher(dir_conf, ignore_status = c("running", "error"), verbose = TRUE) 94 | 95 | log <- read.delim(paste0(dir_conf, "/log_launcher.txt"), header = FALSE) 96 | 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /man/module_configure_task.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/module_configure_task.R 3 | \name{configure_task_server} 4 | \alias{configure_task_server} 5 | \title{Module shiny to configure a task.} 6 | \usage{ 7 | configure_task_server( 8 | input, 9 | output, 10 | session, 11 | btn, 12 | dir_path, 13 | fun_path, 14 | fun_name, 15 | conf_descr = NULL, 16 | fun_args = NULL, 17 | priority = 0L, 18 | compress = TRUE, 19 | labels = list(success = "Task configured !", error = 20 | "Error when configuring the task") 21 | ) 22 | } 23 | \arguments{ 24 | \item{input}{shiny input} 25 | 26 | \item{output}{shiny input} 27 | 28 | \item{session}{shiny input} 29 | 30 | \item{btn}{\code{reactive}. Link to a actionButton for call \code{configure_task} function} 31 | 32 | \item{dir_path}{\code{character}. Tasks location (parent directory).} 33 | 34 | \item{fun_path}{\code{character}. Path to the script of the function.} 35 | 36 | \item{fun_name}{\code{character}. Name of the function in fun_path script.} 37 | 38 | \item{conf_descr}{\code{named list} (NULL). Optional description fields.} 39 | 40 | \item{fun_args}{\code{named list} (NULL). Args of the function, must all be named.} 41 | 42 | \item{priority}{\code{numeric} (0L). Number used to define which task should be launched first.} 43 | 44 | \item{compress}{\code{logical or character} (TRUE). Either a logical specifying whether or not to use "gzip" compression, or one of "gzip", "bzip2" or "xz" to indicate the type of compression to be used.} 45 | 46 | \item{labels}{\code{list}. Modal dialog title.} 47 | } 48 | \value{ 49 | Nothing. 50 | } 51 | \description{ 52 | Module shiny to configure a task. 53 | } 54 | \examples{ 55 | 56 | \donttest{ 57 | 58 | if(interactive()){ 59 | 60 | require(shiny) 61 | 62 | # create temporary directory for conf 63 | dir_conf <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 64 | dir.create(dir_conf, recursive = TRUE) 65 | 66 | # ex fun 67 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 68 | fun_name = "sb_fun_ex" 69 | 70 | # create and save conf 71 | ui <- shiny::fluidPage( 72 | fluidRow( 73 | column(offset = 3, 6, 74 | shiny::actionButton("conf_task", "Configure the task", width = "100\%") 75 | ) 76 | ) 77 | ) 78 | server <- function(input, output, session) { 79 | callModule(configure_task_server, "my_id_1", 80 | btn = reactive(input$conf_task), 81 | dir_path = dir_conf, 82 | conf_descr = list(title = "my_title", 83 | description = "my_descr"), 84 | fun_path = fun_path, 85 | fun_name = fun_name, 86 | fun_args = list(x = 1, 87 | y = 0:4, 88 | z = iris), 89 | priority = 1) 90 | } 91 | shiny::shinyApp(ui = ui, server = server) 92 | 93 | # catch results 94 | list.files(path <- list.dirs(dir_conf, full.names = TRUE, recursive = FALSE)) 95 | path 96 | read_conf <- yaml::read_yaml(paste0(path[1], "/", "conf.yml")) 97 | y <- readRDS(paste0(path[1], "/", "inputs/y.RDS"));y 98 | z <- readRDS(paste0(path[1], "/", "inputs/z.RDS"));z 99 | 100 | } 101 | } 102 | 103 | } 104 | \seealso{ 105 | \code{\link[shinybatch]{tasks_overview_server}} 106 | } 107 | -------------------------------------------------------------------------------- /man/module_tasks_overview.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/module_tasks_overview.R 3 | \name{tasks_overview_server} 4 | \alias{tasks_overview_server} 5 | \alias{tasks_overview_UI} 6 | \title{Module to visualize all the created tasks.} 7 | \usage{ 8 | tasks_overview_server( 9 | input, 10 | output, 11 | session, 12 | dir_path, 13 | allowed_run_info_cols = c("date_creation", "date_start", "date_end", "priority", 14 | "status"), 15 | allow_descr = TRUE, 16 | allowed_function_cols = c("path", "name"), 17 | allow_args = TRUE, 18 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 19 | allow_log_btn = TRUE, 20 | allow_rm_task = TRUE, 21 | update_mode = c("reactive", "button"), 22 | intervalMillis = 5000, 23 | table_fun = function(x) x, 24 | return_value = c("selected_task", "tasks_table"), 25 | datatable_options = list(), 26 | ... 27 | ) 28 | 29 | tasks_overview_UI( 30 | id, 31 | labels = list(btn = "Display/update the tasks", empty_global = 32 | "Empty table of global features.", empty_individual = 33 | "Empty table of individual features.", error_dir_access = 34 | "Cannot access given directory.", logs = "Show logs", remove_task = 35 | "Remove this task ?") 36 | ) 37 | } 38 | \arguments{ 39 | \item{input}{shiny input} 40 | 41 | \item{output}{shiny input} 42 | 43 | \item{session}{shiny input} 44 | 45 | \item{dir_path}{\code{character}. Where to find the tasks directorys.} 46 | 47 | \item{allowed_run_info_cols}{\code{character} (c("date_creation", "date_start", "date_end", "priority", "status")). Run info elements to be kept.} 48 | 49 | \item{allow_descr}{\code{boolean or character} (TRUE). Either a boolean specifying whether or not to keep descr elements, or column names.} 50 | 51 | \item{allowed_function_cols}{\code{character} (c("names", "path")). Function elements to be kept.} 52 | 53 | \item{allow_args}{\code{boolean or character} (TRUE). Either a boolean specifying whether or not to keep args elements, or column names.} 54 | 55 | \item{allowed_status}{\code{character} (c("waiting", "running", "finished", "timeout", "error")). Vector of allowed status.} 56 | 57 | \item{allow_log_btn}{\code{boolean} (TRUE). Whether or not to display a button to show log in modal.} 58 | 59 | \item{allow_rm_task}{\code{boolean} (TRUE). Whether or not to display a button to show log in modal.} 60 | 61 | \item{update_mode}{\code{character} : "reactive" (default) use \code{shiny::reactivePoll} to see if tasks info have changed. 62 | "button" add and use a \code{shiny::actionButton} to update task info table.} 63 | 64 | \item{intervalMillis}{\code{integer}. In case of "reactive" update_mode, time betweens calls} 65 | 66 | \item{table_fun}{\code{function} (function(x) x). Function to be applied on the summary table, making it easy to customize it. First arg must be the summmary table.} 67 | 68 | \item{return_value}{\code{character} : "selected_task" (default) return the selected task using \code{tasks_overview_UI}, 69 | else "tasks_table"can be used to get all tasks information and define a custom UI.} 70 | 71 | \item{datatable_options}{\code{list}. \code{DT::datatable} options argument.} 72 | 73 | \item{...}{\code{}. Additional args to be given to the table_fun function.} 74 | 75 | \item{id}{\code{character}. shiny id to allow multiple instanciation.} 76 | 77 | \item{labels}{\code{list} UI labels} 78 | } 79 | \value{ 80 | if \code{return_value = "selected_task"}, the status of the selected line (one run) of the summary 81 | table and the path to the directory in which its output is stored. if \code{return_value = "tasks_table"}, 82 | all tasks information table 83 | } 84 | \description{ 85 | Module to visualize all the created tasks. 86 | } 87 | \examples{ 88 | 89 | \donttest{ 90 | 91 | 92 | if(interactive()){ 93 | 94 | require(shiny) 95 | 96 | # create temporary directory for conf 97 | dir_conf <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 98 | dir.create(dir_conf, recursive = TRUE) 99 | 100 | # ex fun 101 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 102 | fun_name = "sb_fun_ex" 103 | 104 | # create 2 confs 105 | conf_1 <- configure_task(dir_path = dir_conf, 106 | conf_descr = list(title = "my_title_1", 107 | description = "my_descr_1"), 108 | fun_path = fun_path, 109 | fun_name = fun_name,, 110 | fun_args = list(x = 1, 111 | y = 0:4, 112 | z = iris), 113 | priority = 1) 114 | conf_2 <- configure_task(dir_path = dir_conf, 115 | conf_descr = list(title = "my_title_2", 116 | description = "my_descr_2"), 117 | fun_path = fun_path, 118 | fun_name = fun_name, 119 | fun_args = list(x = 1, 120 | y = 0:4, 121 | z = iris), 122 | priority = 2) 123 | 124 | run_task(paste0(conf_2$dir, "conf.yml")) 125 | 126 | # with package ui 127 | ui <- shiny::fluidPage( 128 | tasks_overview_UI("my_id_1"), 129 | hr(), 130 | verbatimTextOutput("info") 131 | ) 132 | 133 | server <- function(input, output, session) { 134 | selected_task <- callModule(tasks_overview_server, "my_id_1", 135 | dir_path = dir_conf, 136 | allowed_status = c("waiting", "running", "finished", "timeout", "error"), 137 | allowed_run_info_cols = NULL, 138 | allowed_function_cols = "", 139 | allow_descr = TRUE, 140 | allow_args = TRUE, 141 | table_fun = function(x, y) x[, new_col := y], 142 | y = "created using arg. 'table_fun'") 143 | 144 | output$info <- renderPrint({ 145 | selected_task() 146 | }) 147 | } 148 | shiny::shinyApp(ui = ui, server = server) 149 | 150 | # using custom ui 151 | ui <- shiny::fluidPage( 152 | verbatimTextOutput("info") 153 | # and so define what you want ! 154 | ) 155 | 156 | server <- function(input, output, session) { 157 | all_tasks_info <- callModule(tasks_overview_server, "my_id_1", 158 | dir_path = dir_conf, 159 | return_value = "tasks_table" 160 | ) 161 | 162 | output$info <- renderPrint({ 163 | all_tasks_info() 164 | }) 165 | } 166 | shiny::shinyApp(ui = ui, server = server) 167 | 168 | } 169 | } 170 | 171 | } 172 | \seealso{ 173 | \code{\link[shinybatch]{configure_task_server}} 174 | } 175 | -------------------------------------------------------------------------------- /man/run_order.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/run_order.R 3 | \name{run_order} 4 | \alias{run_order} 5 | \title{Compute the order of priority to run a list of configurations} 6 | \usage{ 7 | run_order( 8 | confs, 9 | ignore_status = c("running", "finished", "timeout", "error"), 10 | delay_reruns = TRUE 11 | ) 12 | } 13 | \arguments{ 14 | \item{confs}{\code{list}. List of conf tables.} 15 | 16 | \item{ignore_status}{\code{character} (c("running", "finished", "timeout", "error")). Status to be ignored when launching tasks.} 17 | 18 | \item{delay_reruns}{\code{boolean} (TRUE). When "running", "finished", "timeout" or "error" are not in ignore_status, use the date of the last run instead of 19 | the date of creation of the task to compute the order of (re)run for these tasks. The priority still applies.} 20 | } 21 | \value{ 22 | the order of priority of the given confs according to their priority argument and 23 | date of creation. 24 | } 25 | \description{ 26 | Compute the order of priority to run a list of configurations 27 | } 28 | \details{ 29 | The order is determined first by the priority and then by the date of creation of the task. 30 | } 31 | -------------------------------------------------------------------------------- /man/run_task.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/run_task.R 3 | \name{run_task} 4 | \alias{run_task} 5 | \title{Run the task defined in a conf file} 6 | \usage{ 7 | run_task( 8 | conf_path, 9 | ignore_status = c("running", "finished", "timeout", "error"), 10 | save_rds = TRUE, 11 | compress = TRUE, 12 | return = TRUE, 13 | verbose = FALSE 14 | ) 15 | } 16 | \arguments{ 17 | \item{conf_path}{\code{character}. Path to the conf file.} 18 | 19 | \item{ignore_status}{\code{character} (c("running", "finished", "timeout", "error")). Status to be ignored when launching tasks.} 20 | 21 | \item{save_rds}{\code{logical} Save output in output/res.RDS ? Default to TRUE} 22 | 23 | \item{compress}{\code{logical or character} (TRUE). Either a logical specifying whether or not to use "gzip" compression, or one of "gzip", "bzip2" or "xz" to indicate the type of compression to be used.} 24 | 25 | \item{return}{\code{logical} Get back result in R ? Default to TRUE} 26 | 27 | \item{verbose}{\code{logical} See running task message ? Default to FALSE} 28 | } 29 | \value{ 30 | the result of the task (function applied on prepared args). 31 | } 32 | \description{ 33 | Run the task defined in a conf file 34 | } 35 | \examples{ 36 | 37 | \donttest{ 38 | 39 | # create temporary directory for conf 40 | dir_conf <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 41 | dir.create(dir_conf, recursive = TRUE) 42 | 43 | # ex fun 44 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 45 | fun_name = "sb_fun_ex" 46 | 47 | # create and save conf 48 | conf <- configure_task(dir_path = dir_conf, 49 | conf_descr = list( 50 | title = "my_title", 51 | description = "my_descr" 52 | ), 53 | fun_path = fun_path, 54 | fun_name = fun_name, 55 | fun_args = list( 56 | x = 1, 57 | y = 0:4, 58 | z = iris 59 | ), 60 | priority = 1) 61 | 62 | conf_init <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 63 | y <- readRDS(paste0(conf$dir, "inputs/y.RDS")) 64 | z <- readRDS(paste0(conf$dir, "inputs/z.RDS")) 65 | 66 | run_task(paste0(conf$dir, "conf.yml")) 67 | 68 | # catch results 69 | list.files(conf$dir) 70 | conf_update <- yaml::read_yaml(paste0(conf$dir, "conf.yml")) 71 | output <- readRDS(paste0(conf$dir, "output/res.RDS")) 72 | log <- read.delim(list.files(paste0(conf$dir, "output/"), 73 | pattern = "log_run", full.names = TRUE), header = FALSE) 74 | 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /man/scheduler_shinybatch.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/cron_funs.R 3 | \name{scheduler_init} 4 | \alias{scheduler_init} 5 | \alias{scheduler_add} 6 | \alias{scheduler_remove} 7 | \alias{scheduler_exist} 8 | \alias{scheduler_ls} 9 | \title{Scheduler management of the launcher function} 10 | \usage{ 11 | scheduler_init( 12 | dir_scheduler, 13 | dir_conf, 14 | max_runs = 1, 15 | ignore_status = c("running", "finished", "timeout", "error"), 16 | delay_reruns = TRUE, 17 | filename = paste0("sb_", format(Sys.time(), format = "\%Y\%m\%d"), ".R"), 18 | head_rows = NULL, 19 | timeout = Inf 20 | ) 21 | 22 | scheduler_add( 23 | dir_scheduler, 24 | dir_conf, 25 | max_runs = 1, 26 | ignore_status = c("running", "finished", "timeout", "error"), 27 | delay_reruns = TRUE, 28 | taskname = paste0("sb_", format(Sys.time(), format = "\%Y\%m\%d")), 29 | filename = paste0(taskname, ".R"), 30 | create_file = TRUE, 31 | head_rows = NULL, 32 | timeout = Inf, 33 | ... 34 | ) 35 | 36 | scheduler_remove(taskname, ...) 37 | 38 | scheduler_exist(taskname) 39 | 40 | scheduler_ls(...) 41 | } 42 | \arguments{ 43 | \item{dir_scheduler}{\code{character}. Where to create R scheduler script.} 44 | 45 | \item{dir_conf}{\code{character}. launcher arg : where to find the tasks directories.} 46 | 47 | \item{max_runs}{\code{integer} (1). launcher arg : maximum number of simultaneous running tasks.} 48 | 49 | \item{ignore_status}{\code{character} (c("running", "finished", "timeout", "error")). launcher arg : status to be ignored when launching tasks.} 50 | 51 | \item{delay_reruns}{\code{boolean} (TRUE). When "running", "finished", "timeout" or "error" are not in ignore_status, use the date of the last run instead of 52 | the date of creation of the task to compute the order of (re)run for these tasks. The priority still applies.} 53 | 54 | \item{filename}{\code{character} a character string with the name of the rscript file.} 55 | 56 | \item{head_rows}{\code{character} (NULL). Custom head rows to replace the default ones.} 57 | 58 | \item{timeout}{\code{numeric} Minute. Long task running more than \code{timeout} (perhaps killed with server restart or full memory) are set to "timeout" to enable running other waiting tasks} 59 | 60 | \item{taskname}{\code{character} a character string with the name of the task. (id in Linux cronR, taskname in windows taskscheduleR)} 61 | 62 | \item{create_file}{\code{boolean} (TRUE). Whether or not to also create the R scheduler script with scheduler_init ?} 63 | 64 | \item{...}{\code{}. Additional arguments passed to \code{cronR::cron_add}, \code{cronR::cron_rm}, \code{cronR::cron_ls} (Linux) or \code{taskscheduleR::taskscheduler_create} (Windows).} 65 | } 66 | \value{ 67 | NULL. 68 | } 69 | \description{ 70 | Scheduler management of the launcher function 71 | } 72 | \details{ 73 | Without any frequency argument, default is set to every minute 74 | } 75 | \examples{ 76 | 77 | \dontrun{ 78 | 79 | # Not run: 80 | # create temporary directory for conf 81 | dir_conf <- paste0(tempdir(), "/conf", round(runif(n = 1, max = 10000))) 82 | dir.create(dir_conf, recursive = TRUE) 83 | 84 | # create example of files to be called by the scheduler 85 | # (this fun is called in scheduler_add) 86 | scheduler_init( 87 | dir_scheduler = tempdir(), 88 | dir_conf = dir_conf, 89 | filename = "cron_script.R", 90 | head_rows = NULL 91 | ) 92 | read.delim(paste0(tempdir(), "/cron_script.R"), header = FALSE) 93 | 94 | scheduler_init(dir_scheduler = tempdir(), 95 | dir_conf = dir_conf, 96 | filename = "cron_script.R", 97 | head_rows = c("My_head_row_1", "My_head_row_2")) 98 | read.delim(paste0(tempdir(), "/cron_script.R"), header = FALSE) 99 | 100 | 101 | # start a cron 102 | # create confs to check that it works on it 103 | 104 | # ex fun 105 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 106 | fun_name = "sb_fun_ex" 107 | 108 | # create 2 confs 109 | conf_1 <- configure_task(dir_path = dir_conf, 110 | conf_descr = list( 111 | title_1 = "my_title_1", 112 | description_1 = "my_descr_1" 113 | ), 114 | fun_path = fun_path, 115 | fun_name = fun_name, 116 | fun_args = list( 117 | x = 0, 118 | y = 0:4, 119 | z = iris 120 | ), 121 | priority = 1) 122 | conf_2 <- configure_task(dir_path = dir_conf, 123 | conf_descr = list( 124 | title_2 = "my_title_2", 125 | description_2 = "my_descr_2" 126 | ), 127 | fun_path = fun_path, 128 | fun_name = fun_name, 129 | fun_args = list( 130 | x = 1, 131 | y = 0:4, 132 | z = iris 133 | ), 134 | priority = 2) 135 | 136 | # on LINUX -> Needs cronR package 137 | # on Windows -> Needs taskscheduleR package 138 | 139 | scheduler_add(dir_scheduler = tempdir(), 140 | dir_conf, 141 | max_runs = 1, 142 | ignore_status = c("running", "finished", "timeout", "error"), 143 | delay_reruns = TRUE, 144 | create_file = TRUE, 145 | head_rows = NULL, 146 | taskname = "cron_script_ex") 147 | 148 | scheduler_ls() # display existing crons 149 | 150 | # wait up to 1 min for conf_2 and up to 2 mins for conf_1 151 | yaml::read_yaml(paste0(conf_1$dir, "/conf.yml"))$run_info$status 152 | yaml::read_yaml(paste0(conf_2$dir, "/conf.yml"))$run_info$status 153 | 154 | # check if cron existed 155 | scheduler_exist(taskname = "cron_script_ex") 156 | 157 | # kill selected cron 158 | scheduler_remove(taskname = "cron_script_ex") 159 | scheduler_exist(taskname = "cron_script_ex") 160 | 161 | } 162 | 163 | } 164 | -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | library(shinybatch) 3 | 4 | test_check("shinybatch") 5 | -------------------------------------------------------------------------------- /tests/testthat/test_configure_task.R: -------------------------------------------------------------------------------- 1 | context("test_configure_task") 2 | 3 | # create temporary directory 4 | dir <- tempdir() 5 | 6 | # create and save con 7 | time <- Sys.time() 8 | conf <- configure_task(dir_path = dir, 9 | conf_descr = list(title = "my_title", 10 | description = "my_descr"), 11 | fun_path = dir, # as an example 12 | fun_name = "my_fun_name", 13 | fun_args = list(x = 1, 14 | y = 0:4, 15 | z = iris), 16 | priority = 1) 17 | 18 | 19 | test_that("test outputs", { 20 | # output files 21 | conf_run_info <- conf$run_info 22 | conf_run_info$date_creation <- NULL 23 | expect_equal(conf_run_info, list( 24 | "date_start" = "N/A", 25 | "date_end" = "N/A", 26 | "priority" = 1, 27 | "status" = "waiting") 28 | ) 29 | 30 | expect_equal(conf$descriptive, list(title = "my_title", description = "my_descr")) 31 | expect_equal(conf$`function`, list( 32 | "path" = gsub("\\", "/", dir, fixed = T), 33 | "name" = "my_fun_name") 34 | ) 35 | 36 | expect_equal(c("x", "y", "z"), names(conf$args)) 37 | 38 | expect_equal(list.files(conf$dir), c("conf.yml", "inputs")) 39 | 40 | expect_equal(readRDS(paste0(conf$dir, "inputs/y.RDS")), 0:4) 41 | 42 | expect_equal(readRDS(paste0(conf$dir, "inputs/z.RDS")), iris) 43 | 44 | }) -------------------------------------------------------------------------------- /tests/testthat/test_launcher.R: -------------------------------------------------------------------------------- 1 | context("test_launcher") 2 | 3 | # create temporary directory for conf 4 | dir_conf <- paste0(tempdir(), "/conf_launcher") 5 | if (dir.exists(dir_conf)) unlink(dir_conf, recursive = TRUE) 6 | dir.create(dir_conf, recursive = T) 7 | 8 | # ex fun 9 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 10 | fun_name = "sb_fun_ex" 11 | 12 | # create 2 confs 13 | conf_1 <- configure_task(dir_path = dir_conf, 14 | conf_descr = list(title_1 = "my_title_1", 15 | description_1 = "my_descr_1"), 16 | fun_path = fun_path, 17 | fun_name = fun_name, 18 | fun_args = list(x = 0, 19 | y = 0:4, 20 | z = iris), 21 | priority = 1) 22 | 23 | Sys.sleep(2) 24 | 25 | conf_2 <- configure_task(dir_path = dir_conf, 26 | conf_descr = list(title_2 = "my_title_2", 27 | description_2 = "my_descr_2"), 28 | fun_path = fun_path, 29 | fun_name = fun_name, 30 | fun_args = list(x = 1, 31 | y = 0:4, 32 | z = iris), 33 | priority = 2) 34 | 35 | 36 | test_that("test outputs", { 37 | 38 | expect_error(launcher(NULL)) 39 | 40 | # launch highest priority 41 | expect_equal(launcher(dir_conf), 1) 42 | 43 | Sys.sleep(2) # time to launch the batch script 44 | 45 | if(file.exists(paste0(conf_2$dir, "output/res.RDS"))){ 46 | expect_equal(readRDS(paste0(conf_2$dir, "output/res.RDS")), 47 | 1:5) 48 | expect_equal(yaml::read_yaml(paste0(conf_1$dir, "/conf.yml"))$run_info$status, 49 | "waiting") 50 | expect_equal(yaml::read_yaml(paste0(conf_2$dir, "/conf.yml"))$run_info$status, 51 | "finished") 52 | } 53 | 54 | # launch last conf file 55 | expect_equal(launcher(dir_conf), 56 | 1) 57 | 58 | Sys.sleep(2) # time to launch the batch script 59 | 60 | if(file.exists(paste0(conf_1$dir, "output/res.RDS"))){ 61 | expect_equal(readRDS(paste0(conf_1$dir, "output/res.RDS")), 62 | 0:4) 63 | expect_equal(yaml::read_yaml(paste0(conf_1$dir, "/conf.yml"))$run_info$status, 64 | "finished") 65 | 66 | } 67 | # launch nothing 68 | expect_equal(launcher(dir_conf), 69 | 0) 70 | 71 | # launch 2 confs 72 | # create temporary directory for conf 73 | unlink(tempdir(), recursive = TRUE) 74 | dir.create(dir_conf, recursive = T) 75 | 76 | # ex fun 77 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 78 | fun_name = "sb_fun_ex" 79 | 80 | # create 2 confs 81 | conf_1 <- configure_task(dir_path = dir_conf, 82 | conf_descr = list(title_1 = "my_title_1", 83 | description_1 = "my_descr_1"), 84 | fun_path = fun_path, 85 | fun_name = fun_name, 86 | fun_args = list(x = 0, 87 | y = 0:4, 88 | z = iris), 89 | priority = 1) 90 | 91 | Sys.sleep(2) 92 | 93 | conf_2 <- configure_task(dir_path = dir_conf, 94 | conf_descr = list(title_2 = "my_title_2", 95 | description_2 = "my_descr_2"), 96 | fun_path = fun_path, 97 | fun_name = fun_name, 98 | fun_args = list(x = 1, 99 | y = 0:4, 100 | z = iris), 101 | priority = 2) 102 | 103 | expect_equal(launcher(dir_conf, 104 | max_runs = 2), 105 | 2) 106 | 107 | Sys.sleep(2) # time to launch the batch script 108 | 109 | if(file.exists(paste0(conf_1$dir, "output/res.RDS"))){ 110 | expect_equal(readRDS(paste0(conf_1$dir, "output/res.RDS")), 111 | 0:4) 112 | expect_equal(yaml::read_yaml(paste0(conf_1$dir, "/conf.yml"))$run_info$status, 113 | "finished") 114 | } 115 | 116 | if(file.exists(paste0(conf_2$dir, "output/res.RDS"))){ 117 | expect_equal(readRDS(paste0(conf_2$dir, "output/res.RDS")), 118 | 1:5) 119 | expect_equal(yaml::read_yaml(paste0(conf_2$dir, "/conf.yml"))$run_info$status, 120 | "finished") 121 | } 122 | 123 | }) 124 | -------------------------------------------------------------------------------- /tests/testthat/test_run_order.R: -------------------------------------------------------------------------------- 1 | context("test_run_order") 2 | 3 | 4 | test_that("test outputs", { 5 | 6 | # check date 7 | conf_1 <- configure_task(dir_path = tempdir(), 8 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 9 | priority = 1) 10 | Sys.sleep(1) 11 | conf_2 <- configure_task(dir_path = tempdir(), 12 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 13 | priority = 1) 14 | expect_equal(run_order(list(conf_1, conf_2)), 15 | c(1, 2)) 16 | conf_2 <- configure_task(dir_path = tempdir(), 17 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 18 | priority = 1) 19 | Sys.sleep(1) 20 | conf_1 <- configure_task(dir_path = tempdir(), 21 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 22 | priority = 1) 23 | expect_equal(run_order(list(conf_1, conf_2)), 24 | c(2, 1)) 25 | 26 | # check priority 27 | conf_1 <- configure_task(dir_path = tempdir(), 28 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 29 | priority = 1) 30 | Sys.sleep(1) 31 | conf_2 <- configure_task(dir_path = tempdir(), 32 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 33 | priority = 2) 34 | expect_equal(run_order(list(conf_1, conf_2)), 35 | c(2, 1)) 36 | conf_1 <- configure_task(dir_path = tempdir(), 37 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 38 | priority = 2) 39 | Sys.sleep(1) 40 | conf_2 <- configure_task(dir_path = tempdir(), 41 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 42 | priority = 1) 43 | expect_equal(run_order(list(conf_1, conf_2)), 44 | c(1, 2)) 45 | 46 | # check ignore_status (all except waiting) 47 | conf_1 <- configure_task(dir_path = tempdir(), 48 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 49 | priority = 2) 50 | conf_1$run_info$status <- "error" 51 | Sys.sleep(1) 52 | conf_2 <- configure_task(dir_path = tempdir(), 53 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 54 | priority = 1) 55 | expect_equal(run_order(list(conf_1, conf_2)), 56 | c(2, 1)) 57 | conf_2 <- configure_task(dir_path = tempdir(), 58 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 59 | priority = 2) 60 | conf_2$run_info$status <- "error" 61 | Sys.sleep(1) 62 | conf_1 <- configure_task(dir_path = tempdir(), 63 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 64 | priority = 1) 65 | expect_equal(run_order(list(conf_1, conf_2)), 66 | c(1, 2)) 67 | 68 | # check ignore_status (all except waiting and error) 69 | conf_1 <- configure_task(dir_path = tempdir(), 70 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 71 | priority = 2) 72 | conf_1$run_info$status <- "error" 73 | Sys.sleep(1) 74 | conf_2 <- configure_task(dir_path = tempdir(), 75 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 76 | priority = 1) 77 | expect_equal(run_order(list(conf_1, conf_2), 78 | ignore_status = c("running", "finished")), 79 | c(1, 2)) 80 | conf_2 <- configure_task(dir_path = tempdir(), 81 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 82 | priority = 2) 83 | conf_2$run_info$status <- "error" 84 | Sys.sleep(1) 85 | conf_1 <- configure_task(dir_path = tempdir(), 86 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 87 | priority = 1) 88 | expect_equal(run_order(list(conf_1, conf_2), 89 | ignore_status = c("running", "finished")), 90 | c(2, 1)) 91 | 92 | # check delay_reruns (T) 93 | conf_1 <- configure_task(dir_path = tempdir(), 94 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 95 | priority = 1) 96 | conf_1$run_info$status <- "error" 97 | Sys.sleep(1) 98 | conf_2 <- configure_task(dir_path = tempdir(), 99 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 100 | priority = 1) 101 | Sys.sleep(1) 102 | conf_1$run_info$date_start <- as.character(Sys.time()) 103 | expect_equal(run_order(list(conf_1, conf_2), 104 | ignore_status = c("running", "finished"), 105 | delay_reruns = T), 106 | c(2, 1)) 107 | conf_2 <- configure_task(dir_path = tempdir(), 108 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 109 | priority = 1) 110 | conf_2$run_info$status <- "error" 111 | Sys.sleep(1) 112 | conf_1 <- configure_task(dir_path = tempdir(), 113 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 114 | priority = 1) 115 | Sys.sleep(1) 116 | conf_2$run_info$date_start <- as.character(Sys.time()) 117 | expect_equal(run_order(list(conf_1, conf_2), 118 | ignore_status = c("running", "finished"), 119 | delay_reruns = T), 120 | c(1, 2)) 121 | 122 | # check delay_reruns (F) 123 | conf_1 <- configure_task(dir_path = tempdir(), 124 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 125 | priority = 1) 126 | conf_1$run_info$status <- "error" 127 | Sys.sleep(1) 128 | conf_2 <- configure_task(dir_path = tempdir(), 129 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 130 | priority = 1) 131 | Sys.sleep(1) 132 | conf_1$run_info$date_start <- as.character(Sys.time()) 133 | expect_equal(run_order(list(conf_1, conf_2), 134 | ignore_status = c("running", "finished"), 135 | delay_reruns = F), 136 | c(1, 2)) 137 | conf_2 <- configure_task(dir_path = tempdir(), 138 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 139 | priority = 1) 140 | conf_2$run_info$status <- "error" 141 | Sys.sleep(1) 142 | conf_1 <- configure_task(dir_path = tempdir(), 143 | fun_path = tempdir(), fun_name = "fun_name", fun_args = list(arg_1 = NA), 144 | priority = 1) 145 | Sys.sleep(1) 146 | conf_2$run_info$date_start <- as.character(Sys.time()) 147 | expect_equal(run_order(list(conf_1, conf_2), 148 | ignore_status = c("running", "finished"), 149 | delay_reruns = F), 150 | c(2, 1)) 151 | 152 | }) -------------------------------------------------------------------------------- /tests/testthat/test_run_task.R: -------------------------------------------------------------------------------- 1 | context("test_run_task") 2 | 3 | # create temporary directory for conf 4 | dir_conf <- paste0(tempdir(), "/conf") 5 | if (dir.exists(dir_conf)) unlink(dir_conf, recursive = TRUE) 6 | dir.create(dir_conf, recursive = T) 7 | 8 | # ex fun 9 | fun_path = system.file("ex_fun/sb_fun_ex.R", package = "shinybatch") 10 | fun_name = "sb_fun_ex" 11 | 12 | # create and save conf 13 | conf <- configure_task(dir_path = dir_conf, 14 | conf_descr = list(title = "my_title", 15 | description = "my_descr"), 16 | fun_path = fun_path, 17 | fun_name = fun_name, 18 | fun_args = list(x = 1, 19 | y = 0:4, 20 | z = iris), 21 | priority = 1) 22 | 23 | 24 | test_that("test outputs", { 25 | 26 | # fun output 27 | expect_equal(suppressWarnings(run_task(paste0(conf$dir, "conf.yml"))), 1:5) 28 | 29 | # conf updates 30 | ##### 31 | # 1 # 32 | ##### 33 | expect_equal(yaml::read_yaml(paste0(conf$dir, "conf.yml"))$run_info$status, "finished") 34 | 35 | ##### 36 | # 2 # 37 | ##### 38 | # create temporary directory for conf 39 | dir_conf <- tempdir() 40 | # create temporary directory for fun 41 | dir_fun <- paste0(tempdir(), "/fun") 42 | if (dir.exists(dir_fun)) unlink(dir_fun, recursive = TRUE) 43 | dir.create(dir_fun) 44 | con <- file(paste0(dir_fun, "/fun_script.R")) 45 | writeLines(c("my_fun <- function(x, y, z) {", 46 | " res <- x + y ;", 47 | " stop('Error') ;", 48 | " res", 49 | "}"), 50 | con) 51 | close(con) 52 | 53 | # create and save conf 54 | conf <- configure_task(dir_path = dir_conf, 55 | conf_descr = list(title = "my_title", 56 | description = "my_descr"), 57 | fun_path = paste0(dir_fun, "/fun_script.R"), 58 | fun_name = "my_fun", 59 | fun_args = list(x = 1, 60 | y = 0:4, 61 | z = iris), 62 | priority = 1) 63 | 64 | time <- Sys.time() 65 | try(run_task(paste0(conf$dir, "conf.yml")), silent = T) 66 | 67 | expect_equal(yaml::read_yaml(paste0(conf$dir, "conf.yml"))$run_info$status, "error") 68 | 69 | }) -------------------------------------------------------------------------------- /vignettes/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.R 3 | -------------------------------------------------------------------------------- /vignettes/lancement_python.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Launch a task in Python with Shinybach" 3 | output: rmarkdown::html_vignette 4 | vignette: > 5 | %\VignetteIndexEntry{lancement_python} 6 | %\VignetteEngine{knitr::rmarkdown} 7 | %\VignetteEncoding{UTF-8} 8 | --- 9 | 10 | ```{r, include = FALSE} 11 | knitr::opts_chunk$set( 12 | collapse = TRUE, 13 | comment = "#>" 14 | ) 15 | ``` 16 | 17 | ```{r setup, include = F} 18 | require(shinybatch) 19 | ``` 20 | 21 | In its basic use, the package runs a R function by : 22 | 23 | - sourcing the R script of the function 24 | - retrieving the function name and args in the yaml conf file 25 | - calling the function by using *do.call()* 26 | 27 | 28 | To run a script that is not written in R, it is possible to use the R function as a wrapper. 29 | 30 |
31 | 32 | We give an example for ``Python``, using package **reticulate**. Another solution would be to use the *system()* function. 33 | This example re-uses the demo app (stored in the package: ``system.file( "demo_app.R", package = "shinybatch")``): 34 | 35 |
36 | 37 | First block (same as the demo app): 38 | 39 | - load packages 40 | - create temporary directory to save the configuration files 41 | - set the path of the R function 42 | - initialize the scheduler 43 | 44 | ```{r, eval = F} 45 | require(shiny) 46 | require(shinydashboard) 47 | require(DT) 48 | require(shinybatch) 49 | 50 | # create directory for conf 51 | dir_conf <- paste0(tempdir(), "/conf") 52 | if (dir.exists(dir_conf)) unlink(dir_conf, recursive = TRUE) 53 | dir.create(dir_conf, recursive = T) 54 | 55 | # get path of the wrapper of th python fun 56 | wrapper_path <- system.file("/ex_fun/sb_fun_demo_app_python.R", package = "shinybatch") 57 | 58 | # init shceduler 59 | dir_scheduler <- paste0(tempdir(), "/cron_demo_python") 60 | if (dir.exists(dir_scheduler)) unlink(dir_scheduler, recursive = TRUE) 61 | dir.create(dir_scheduler, recursive = T) 62 | 63 | scheduler_add(dir_scheduler = dir_scheduler, 64 | dir_conf = dir_conf, 65 | max_runs = 1, 66 | head_rows = NULL, 67 | taskname = "cr_python_demo") 68 | ``` 69 | 70 | 71 | The wrapper function: 72 | 73 | - The Python function can be run as is 74 | - the arguments should be provided in R and then cast to python in the wrapper or in the Python fun 75 | 76 | The Python function applies a single regexpr to change 'input' to 'output'. 77 | ```{r, eval = F} 78 | wrapper_py <- function(vect) { 79 | require(reticulate) 80 | 81 | env <- py_run_file(system.file("/ex_fun/sb_fun_demo_app_python.py", package = "shinybatch")) ; 82 | env$apply_regexprs(vect = vect, 83 | regexprs = reticulate::dict(list("input" = "output"))) 84 | } 85 | ``` 86 | 87 |
88 | 89 | Second block: 90 | 91 | - define the Shiny UI 92 | 93 | Modifications: 94 | 95 | - new inputs to fit the usecase 96 | ```{r, eval = F} 97 | # define UI 98 | ui <- shinydashboard::dashboardPage( 99 | shinydashboard::dashboardHeader(title = "Shinybatch"), 100 | shinydashboard::dashboardSidebar(disable = T), 101 | shinydashboard::dashboardBody( 102 | fluidRow( 103 | shinydashboard::tabBox(width = 12, 104 | tabPanel("Configure tasks", 105 | fluidRow( 106 | # conf args 107 | column(2, 108 | numericInput("priority", "Priority", 109 | min = 0, max = 10, value = 1, step = 1) 110 | ), 111 | column(2, 112 | textInput("title", "Title", 113 | value = "Task title", width = "100%") 114 | ), 115 | column(3, 116 | textInput("description", "Description", 117 | value = "Task description", width = "100%") 118 | ), 119 | # fun args 120 | column(3, 121 | textInput("regexpr", "Regular expression", 122 | value = "Test input", width = "100%") 123 | ), 124 | column(2, 125 | numericInput("sleep_time", "Sleep time (s)", 126 | min = 0, max = 30, value = 0, step = 1) 127 | ) 128 | ), 129 | hr(), 130 | fluidRow( 131 | column(6, offset = 3, 132 | actionButton("go_task", "Configure the task", width = "100%") 133 | ) 134 | ) 135 | 136 | ), 137 | tabPanel("View tasks", 138 | fluidRow( 139 | column(12, 140 | tasks_overview_UI("my_id_2") 141 | ) 142 | ), 143 | conditionalPanel(condition = "output.launch_task", 144 | br(), 145 | hr(), 146 | fluidRow( 147 | conditionalPanel(condition = "output.launch_task", 148 | column(12, 149 | div(actionButton("show_task", "Display task result", width = "40%"), 150 | align = "center") 151 | ) 152 | ), 153 | column(12, 154 | conditionalPanel(condition = "input.show_task > 0", 155 | textOutput("task_text") 156 | ) 157 | ) 158 | ) 159 | ) 160 | ) 161 | ) 162 | ) 163 | ) 164 | ) 165 | ``` 166 | 167 |
168 | 169 | Third block: 170 | 171 | - define the Shiny server 172 | 173 | Modifications: 174 | 175 | - ajust *callModule()* to fit the usecase 176 | ```{r, eval = F} 177 | # define server 178 | # call both shinybatch modules + display result 179 | server <- function(input, output, session) { 180 | 181 | # call module to configure a task 182 | # connect app inputs to the module 183 | callModule(configure_task_server, "my_id_1", 184 | btn = reactive(input$go_task), 185 | dir_path = dir_conf, 186 | conf_descr = reactive(list("title" = input$title, 187 | "description" = input$description)), 188 | fun_path = wrapper_path, 189 | fun_name = "wrapper_py", 190 | fun_args = reactive(list(vect = input$regexpr)), 191 | priority = reactive(input$priority)) 192 | 193 | # call module to view tasks 194 | sel_task <- callModule(tasks_overview_server, "my_id_2", 195 | dir_path = dir_conf, 196 | update_mode = "reactive", 197 | allowed_status = c("waiting", "running", "finished", "error"), 198 | allowed_run_info_cols = NULL, 199 | allowed_function_cols = NULL, 200 | allow_descr = T, 201 | allow_args = T) 202 | 203 | # check if sel_task can be displayed 204 | output$launch_task <- reactive({ 205 | ! is.null(sel_task()) && length(sel_task()) > 0 && sel_task()$status == "finished" 206 | }) 207 | outputOptions(output, "launch_task", suspendWhenHidden = FALSE) 208 | 209 | # display task 210 | output$task_text <- renderText({ 211 | cpt <- input$show_task 212 | sel_task <- sel_task() 213 | 214 | isolate({ 215 | if (cpt > 0 && length(sel_task) > 0 && sel_task$status == "finished") { 216 | readRDS(paste0(sel_task$path, "/res.RDS")) 217 | } 218 | }) 219 | }) 220 | } 221 | ``` 222 | 223 |
224 | 225 | Launch the app: 226 | ```{r, eval = F} 227 | # launch app 228 | shiny::shinyApp(ui = ui, 229 | server = server, 230 | onStart = function() { 231 | onStop(function() { 232 | scheduler_remove(taskname = "cr_python_demo") 233 | }) 234 | }) 235 | ``` --------------------------------------------------------------------------------