├── .Rbuildignore ├── .github ├── .gitignore └── workflows │ └── R-CMD-check.yaml ├── .gitignore ├── DESCRIPTION ├── NAMESPACE ├── NEWS.md ├── R ├── addin.R ├── clean_runs.R ├── copy.R ├── flags.R ├── ls_runs.R ├── metrics_viewer.R ├── package.R ├── reexports.R ├── run_dir.R ├── run_metadata.R ├── training_run.R ├── utils.R └── views.R ├── README.md ├── cran-comments.md ├── dev ├── boston_housing │ └── train.R ├── flags.R ├── mnist_mlp │ ├── .gitignore │ └── mnist_mlp.R └── steps_column │ └── view.html ├── images ├── compare_runs.png ├── ls_runs_compare.png ├── ls_runs_rstudio.png ├── rstudio_addin.png ├── rstudio_publish.png ├── rstudio_terminal.png ├── view_run.png └── view_run_output.png ├── inst ├── examples │ └── mnist_mlp │ │ └── mnist_mlp.R ├── rstudio │ └── addins.dcf └── views │ ├── compare_runs.html │ ├── components │ ├── c3.html │ ├── dashboard.html │ ├── diff2html.html │ ├── highlight_js.html │ ├── jquery-AUTHORS.txt │ ├── jquery.html │ ├── material_icons.html │ ├── materialize.html │ ├── metrics_charts.html │ ├── roboto.html │ ├── vue_js.html │ └── vue_min_js.html │ ├── metrics.html │ └── view_run.html ├── man-roxygen └── roxlate-metrics-format.R ├── man ├── as_run_dir.Rd ├── clean_runs.Rd ├── compare_runs.Rd ├── copy_run.Rd ├── flags.Rd ├── is_run_active.Rd ├── latest_run.Rd ├── ls_runs.Rd ├── reexports.Rd ├── run_dir.Rd ├── run_info.Rd ├── save_run_comparison.Rd ├── save_run_view.Rd ├── training_run.Rd ├── tuning_run.Rd ├── unique_run_dir.Rd ├── view_run.Rd ├── view_run_metrics.Rd ├── write_run_data.Rd └── write_run_metadata.Rd ├── pkgdown ├── _pkgdown.yml ├── extra.css └── extra.js ├── tests ├── testthat.R └── testthat │ ├── extra.dat │ ├── flags-learning-rate.yml │ ├── flags-override.yml │ ├── flags-precision.R │ ├── flags-profile-override.yml │ ├── flags.rds │ ├── helper.R │ ├── metrics.rds │ ├── subdir │ └── extra.dat │ ├── test-copy.R │ ├── test-flags.R │ ├── test-run-data.R │ ├── test-runs.R │ ├── test-tuning.R │ ├── train-error.R │ ├── train.R │ └── write_run_data.R ├── tfruns.Rproj └── vignettes ├── images ├── compare_runs.png ├── ls_runs_compare.png ├── ls_runs_rstudio.png ├── rstudio_addin.png ├── rstudio_publish.png ├── rstudio_terminal.png ├── view_run.png └── view_run_output.png ├── managing.Rmd ├── overview.Rmd └── tuning.Rmd /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | ^man-roxygen/ 4 | ^pkgdown$ 5 | ^runs$ 6 | ^dev$ 7 | ^docs$ 8 | ^index\.Rmd$ 9 | ^index\.html$ 10 | ^inst/examples/ 11 | ^tests/testthat/runs/ 12 | ^images$ 13 | ^\.travis\.yml$ 14 | ^\.github$ 15 | ^cran-comments\.md$ 16 | ^CRAN-RELEASE$ 17 | ^CRAN-SUBMISSION$ 18 | -------------------------------------------------------------------------------- /.github/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | -------------------------------------------------------------------------------- /.github/workflows/R-CMD-check.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | pull_request: 6 | # schedule: 7 | # - cron: '0 1 * * *' 8 | 9 | name: R-CMD-check 10 | 11 | jobs: 12 | R-CMD-check: 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: ['windows-latest', 'macOS-latest', 'ubuntu-latest'] 17 | # include: 18 | # - os: ubuntu-18.04 19 | # cran: https://demo.rstudiopm.com/all/__linux__/focal/latest 20 | 21 | 22 | runs-on: ${{ matrix.os }} 23 | name: ${{ matrix.os }} 24 | env: 25 | R_REMOTES_NO_ERRORS_FROM_WARNINGS: true 26 | PIP_NO_WARN_SCRIPT_LOCATION: false 27 | RETICULATE_AUTOCONFIGURE: 'FALSE' 28 | CRAN: ${{ matrix.cran }} 29 | 30 | 31 | steps: 32 | - uses: actions/checkout@v3 33 | 34 | - uses: r-lib/actions/setup-r@v2 35 | 36 | - uses: r-lib/actions/setup-pandoc@v2 37 | 38 | - uses: r-lib/actions/setup-r-dependencies@v2 39 | with: 40 | extra-packages: rcmdcheck remotes reticulate 41 | 42 | # - name: Install system dependencies 43 | # if: runner.os == 'Linux' 44 | # shell: bash 45 | # run: | 46 | # . /etc/os-release 47 | # while read -r cmd 48 | # do 49 | # echo "$cmd" 50 | # sudo $cmd 51 | # done < <(Rscript -e "writeLines(remotes::system_requirements('$ID-$VERSION_ID'))") 52 | 53 | - uses: actions/setup-python@v4 54 | with: 55 | python-version: '3.10' 56 | 57 | - name: setup r-reticulate venv 58 | shell: Rscript {0} 59 | run: | 60 | remotes::install_local() 61 | library(reticulate) 62 | virtualenv_create("r-reticulate", Sys.which("python"), 63 | packages = c( 64 | "tensorflow-cpu" 65 | )) 66 | python <- reticulate::virtualenv_python("r-reticulate") 67 | writeLines(sprintf("RETICULATE_PYTHON=%s", python), 68 | Sys.getenv("GITHUB_ENV")) 69 | 70 | - name: Check 71 | continue-on-error: ${{ contains(matrix.allow_failure, 'true') }} 72 | run: Rscript -e "rcmdcheck::rcmdcheck(args = '--no-manual', error_on = 'warning', check_dir = 'check')" 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | .DS_Store 6 | runs 7 | docs 8 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: tfruns 2 | Type: Package 3 | Title: Training Run Tools for 'TensorFlow' 4 | Version: 1.5.3 5 | Authors@R: c( 6 | person("Tomasz", "Kalinowski", role = c("ctb", "cre"), email = "tomasz@rstudio.com"), 7 | person("Daniel", "Falbel", role = c("ctb"), email = "daniel@rstudio.com"), 8 | person("JJ", "Allaire", role = c("aut"), email = "jj@rstudio.com"), 9 | person(family = "RStudio", role = c("cph", "fnd")), 10 | person("Mike", "Bostock", role = c("cph"), 11 | comment = "D3 library - https://d3js.org/"), 12 | person("Masayuki", "Tanaka", role = c("cph"), 13 | comment = "C3 library - http://c3js.org/"), 14 | person(family = "jQuery Foundation", role = "cph", 15 | comment = "jQuery library"), 16 | person(family = "jQuery contributors", role = c("cph"), 17 | comment = "jQuery library; authors: inst/views/components/jquery-AUTHORS.txt"), 18 | person("Shaun", "Bowe", role = c("cph"), 19 | comment= "jQuery visibilityChanged plugin"), 20 | person(family = "Materialize", role = c("cph"), 21 | comment = "Materizlize library - https://materializecss.com/"), 22 | person("Yuxi", "You", role = c("cph"), 23 | comment = "Vue.js library - https://vuejs.org/"), 24 | person("Kevin", "Decker", role = c("cph"), 25 | comment = "jsdiff library - https://github.com/kpdecker/jsdiff/"), 26 | person("Rodrigo", "Fernandes", role = c("cph"), 27 | comment = "diff2html library - https://diff2html.xyz/"), 28 | person("Ivan", "Sagalaev", role = c("cph"), 29 | comment = "highlight.js library - https://highlightjs.org/"), 30 | person("Yauheni", "Pakala", role = c("cph"), 31 | comment = "highlightjs-line-numbers library") 32 | ) 33 | Description: Create and manage unique directories for each 'TensorFlow' 34 | training run. Provides a unique, time stamped directory for each run 35 | along with functions to retrieve the directory of the latest run or 36 | latest several runs. 37 | License: Apache License 2.0 38 | URL: https://github.com/rstudio/tfruns 39 | BugReports: https://github.com/rstudio/tfruns/issues 40 | Depends: 41 | R (>= 3.1) 42 | Imports: 43 | utils, 44 | jsonlite (>= 1.2), 45 | base64enc, 46 | yaml, 47 | config, 48 | magrittr, 49 | whisker, 50 | tidyselect, 51 | rlang, 52 | rstudioapi (>= 0.7), 53 | reticulate 54 | Suggests: 55 | testthat, 56 | knitr, 57 | withr, 58 | here, 59 | rmarkdown 60 | Encoding: UTF-8 61 | LazyData: true 62 | Roxygen: list(markdown = TRUE) 63 | RoxygenNote: 7.2.1 64 | VignetteBuilder: knitr 65 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | S3method(as.data.frame,tfruns_flags) 4 | S3method(as_run_dir,character) 5 | S3method(as_run_dir,data.frame) 6 | S3method(as_run_dir,list) 7 | S3method(as_run_dir,tfruns_run) 8 | S3method(print,tfruns_flags) 9 | S3method(print,tfruns_model_summary) 10 | S3method(print,tfruns_run) 11 | S3method(print,tfruns_runs_df) 12 | S3method(print,tfruns_viewed_run) 13 | export("%>%") 14 | export(as_run_dir) 15 | export(clean_runs) 16 | export(compare_runs) 17 | export(copy_run) 18 | export(copy_run_files) 19 | export(flag_boolean) 20 | export(flag_integer) 21 | export(flag_numeric) 22 | export(flag_string) 23 | export(flags) 24 | export(is_run_active) 25 | export(latest_run) 26 | export(ls_runs) 27 | export(purge_runs) 28 | export(run_dir) 29 | export(run_info) 30 | export(save_run_comparison) 31 | export(save_run_view) 32 | export(training_run) 33 | export(tuning_run) 34 | export(unique_run_dir) 35 | export(update_run_metrics) 36 | export(view_run) 37 | export(view_run_metrics) 38 | export(write_run_data) 39 | export(write_run_metadata) 40 | import(base64enc) 41 | import(whisker) 42 | importFrom(magrittr,"%>%") 43 | importFrom(utils,str) 44 | -------------------------------------------------------------------------------- /NEWS.md: -------------------------------------------------------------------------------- 1 | # tfruns 1.5.3 2 | 3 | * Updates for R-devel (4.4) 4 | 5 | # tfruns 1.5.2 6 | 7 | * Added support for dark mode in the keras metrics viewer. 8 | 9 | # tfruns 1.5.1 10 | 11 | * Updated Rd docs for compatibility with R-4.2 / HTML5 12 | 13 | # tfruns 1.5.0 14 | 15 | # tfruns 1.4.0.9000 16 | 17 | * Added a `NEWS.md` file to track changes to the package. (#56) 18 | * Added GitHub actions CI and removed Travis. (#54) 19 | * Fixed issue with precision of serialized metrics and flags. (#55) 20 | -------------------------------------------------------------------------------- /R/addin.R: -------------------------------------------------------------------------------- 1 | 2 | add_in_training_run <- function() { 3 | editor_context <- rstudioapi::getSourceEditorContext() 4 | editor_path <- editor_context$path 5 | if (isTRUE(nzchar(editor_path))) { 6 | rstudioapi::documentSaveAll() 7 | normalized_wd <- normalizePath(getwd(), winslash = "/", mustWork = FALSE) 8 | normalized_editor_path <- normalizePath(editor_path, winslash = "/", mustWork = FALSE) 9 | if (grepl(paste0("^", normalized_wd), normalized_editor_path)) 10 | editor_path <- sub(paste0("^", normalized_wd, "[/\\\\]"), "", normalized_editor_path) 11 | rstudioapi::sendToConsole(sprintf('tfruns::training_run("%s")', editor_path)) 12 | } else { 13 | message("Unable to perform training run (active source file is not an R script)") 14 | } 15 | } 16 | 17 | 18 | add_in_view_latest_run <- function() { 19 | rstudioapi::sendToConsole("tfruns::view_run()") 20 | } 21 | 22 | add_in_view_run_history <- function() { 23 | rstudioapi::sendToConsole("View(tfruns::ls_runs())") 24 | } 25 | -------------------------------------------------------------------------------- /R/clean_runs.R: -------------------------------------------------------------------------------- 1 | #' Clean run directories 2 | #' 3 | #' Remove run directories from the filesystem. 4 | #' 5 | #' The `clean_runs()` function moves the specified runs (by default, 6 | #' all runs) into an "archive" subdirectory of the "runs" directory. 7 | #' 8 | #' The `purge_runs()` function permanently deletes the "archive" 9 | #' subdirectory. 10 | #' 11 | #' @inheritParams latest_run 12 | #' @param runs Runs to clean. Can be specified as a data frame 13 | #' (as returned by [ls_runs()]) or as a character vector of 14 | #' run directories. 15 | #' @param confirm `TRUE` to confirm before performing operation 16 | #' 17 | #' @examples \dontrun{ 18 | #' clean_runs(ls_runs(completed == FALSE)) 19 | #' } 20 | #' 21 | #' @family run management 22 | #' 23 | #' @export 24 | clean_runs <- function(runs = ls_runs(runs_dir = runs_dir), 25 | runs_dir = getOption("tfruns.runs_dir", "runs"), 26 | confirm = interactive()) { 27 | 28 | # check for a run list that's been specified (otherwise default to all) 29 | if (!missing(runs)) 30 | run_dirs <- as_run_dir(runs) 31 | else 32 | run_dirs <- list_run_dirs(runs_dir = runs_dir) 33 | 34 | # check for no runs 35 | if (length(run_dirs) == 0) { 36 | message("No runs found to clean.") 37 | return(invisible(NULL)) 38 | } 39 | 40 | # compute archive dir 41 | archive_dir <- file.path(runs_dir, "archive") 42 | 43 | # confirm if requested 44 | if (confirm) { 45 | prompt <- readline(sprintf("Move %d run directories to %s? [Y/n]: ", 46 | length(run_dirs), archive_dir)) 47 | if (nzchar(prompt) && tolower(prompt) != 'y') 48 | return(invisible(NULL)) 49 | } 50 | 51 | # move to the archive directory 52 | if (!utils::file_test("-d", archive_dir)) 53 | dir.create(archive_dir, recursive = TRUE) 54 | file.rename(run_dirs, file.path(archive_dir, basename(run_dirs))) 55 | 56 | # print message 57 | message(sprintf('Moved %d runs to %s (purge_runs() to remove permanently)', 58 | length(run_dirs), archive_dir)) 59 | 60 | # return NULL 61 | invisible(NULL) 62 | } 63 | 64 | #' @rdname clean_runs 65 | #' @export 66 | purge_runs <- function(runs_dir = getOption("tfruns.runs_dir", "runs"), 67 | confirm = interactive()) { 68 | 69 | # enumerate dirs 70 | archive_dir <- file.path(runs_dir, "archive") 71 | run_dirs <- list_run_dirs(runs_dir = archive_dir) 72 | 73 | # prompt 74 | if (confirm) { 75 | prompt <- readline(sprintf("Permanently remove %d run directories from %s? [Y/n]: ", 76 | length(run_dirs), archive_dir)) 77 | if (nzchar(prompt) && tolower(prompt) != 'y') 78 | return(invisible(NULL)) 79 | } 80 | 81 | # remove 82 | unlink(run_dirs, recursive = TRUE) 83 | 84 | # print message 85 | message(sprintf("Permanently removed %d runs", length(run_dirs))) 86 | 87 | # return NULL 88 | invisible(NULL) 89 | } 90 | 91 | 92 | -------------------------------------------------------------------------------- /R/copy.R: -------------------------------------------------------------------------------- 1 | 2 | #' Copy run directories 3 | #' 4 | #' Functions for exporting/copying run directories and run artifact files. 5 | #' 6 | #' @inheritParams run_info 7 | #' 8 | #' @param to Name of parent directory to copy run(s) into. Defaults to the 9 | #' current working directory. 10 | #' 11 | #' @param rename Rename run directory after copying. If not specified this 12 | #' defaults to the basename of the run directory (e.g. 13 | #' "2017-09-24T10-54-00Z"). 14 | #' 15 | #' @details Use `copy_run` to copy one or more run directories. 16 | #' 17 | #' Use `copy_run_files` to copy only files saved/generated by training run 18 | #' scripts (e.g. saved models, checkpoints, etc.). 19 | #' 20 | #' @return Logical vector indicating which operation succeeded for each of the 21 | #' run directories specified. 22 | #' 23 | #' @family run management 24 | #' 25 | #' @examples \dontrun{ 26 | #' 27 | #' # export a run directory to the current working directory 28 | #' copy_run("runs/2017-09-24T10-54-00Z") 29 | #' 30 | #' # export to the current working directory then rename 31 | #' copy_run("runs/2017-09-24T10-54-00Z", rename = "best-run") 32 | #' 33 | #' # export artifact files only to the current working directory then rename 34 | #' copy_run_files("runs/2017-09-24T10-54-00Z", rename = "best-model") 35 | #' 36 | #' # export 3 best eval_acc to a "best-runs" directory 37 | #' copy_run(ls_runs(order = eval_acc)[1:3,], to = "best-runs") 38 | #' 39 | #' } 40 | #' @export 41 | copy_run <- function(run_dir, to = ".", rename = NULL) { 42 | do_copy_run(run_dir, to, rename) 43 | } 44 | 45 | 46 | #' @rdname copy_run 47 | #' @export 48 | copy_run_files <- function(run_dir, to = ".", rename = NULL) { 49 | do_copy_run(run_dir, to, rename, remove_dirs = c("tfruns.d", "logs", "plots")) 50 | } 51 | 52 | do_copy_run <- function(run_dir, to = ".", rename = NULL, remove_dirs = NULL) { 53 | 54 | # resolve run_dir 55 | run_dir <- as_run_dir(run_dir) 56 | 57 | # rename must be the same length as run_dir 58 | if (!is.null(rename) && length(rename) != length(run_dir)) 59 | stop("The rename parameter must have the same length as the run_dir parameter") 60 | 61 | # to must be length 1 62 | if (length(to) != 1) 63 | stop("You must pass a single 'to' directory to copy_run") 64 | 65 | # create to if it doesn't exist 66 | if (!utils::file_test("-d",to)) 67 | dir.create(to, recursive = TRUE) 68 | 69 | # copy to targets 70 | result <- file.copy(run_dir, to, recursive = TRUE, overwrite = TRUE, copy.date = TRUE) 71 | 72 | # note targets (for rename/remove below) 73 | targets <- file.path(to, basename(run_dir)) 74 | 75 | # perform renames if requested 76 | if (!is.null(rename)) { 77 | renames <- file.path(to, rename) 78 | unlink(renames, recursive = TRUE) 79 | result <- file.rename(targets, renames) 80 | targets <- renames 81 | } 82 | 83 | # perform removes if requested 84 | if (!is.null(remove_dirs)) { 85 | for (target in targets) 86 | unlink(file.path(target, remove_dirs), recursive = TRUE) 87 | } 88 | 89 | # return 90 | invisible(result) 91 | } 92 | 93 | 94 | -------------------------------------------------------------------------------- /R/flags.R: -------------------------------------------------------------------------------- 1 | 2 | #' Flags for a training run 3 | #' 4 | #' Define the flags (name, type, default value, description) which paramaterize 5 | #' a training run. Optionally read overrides of the default values from a 6 | #' "flags.yml" config file and/or command line arguments. 7 | #' 8 | #' @param name Flag name 9 | #' @param default Flag default value 10 | #' @param description Flag description 11 | #' @param ... One or more flag definitions 12 | #' @param config The configuration to use. Defaults to the active configuration 13 | #' for the current environment (as specified by the `R_CONFIG_ACTIVE` 14 | #' environment variable), or `default` when unset. 15 | #' @param file The flags YAML file to read 16 | #' @param arguments The command line arguments (as a character vector) to be 17 | #' parsed. 18 | #' 19 | #' @return Named list of training flags 20 | #' 21 | #' @section Config File Flags: 22 | #' 23 | #' Config file flags are defined a YAML configuration file (by default 24 | #' named "flags.yml"). Flags can either appear at the top-level of 25 | #' the YAML or can be inclued in named configuration sections 26 | #' (see the \href{https://github.com/rstudio/config}{config package} for 27 | #' details). 28 | #' 29 | #' @section Command Line Flags: 30 | #' 31 | #' Command line flags should be of the form `--key=value` or 32 | #' `--key value`. The values are assumed to be valid `yaml` and 33 | #' will be converted using [yaml.load()]. 34 | #' 35 | #' @examples 36 | #' \dontrun{ 37 | #' library(tfruns) 38 | #' 39 | #' # define flags and parse flag values from flags.yml and the command line 40 | #' FLAGS <- flags( 41 | #' flag_numeric('learning_rate', 0.01, 'Initial learning rate.'), 42 | #' flag_integer('max_steps', 5000, 'Number of steps to run trainer.'), 43 | #' flag_string('data_dir', 'MNIST-data', 'Directory for training data'), 44 | #' flag_boolean('fake_data', FALSE, 'If true, use fake data for testing') 45 | #' ) 46 | #' } 47 | #' 48 | #' @export 49 | flags <- function(..., 50 | config = Sys.getenv("R_CONFIG_ACTIVE", unset = "default"), 51 | file = "flags.yml", 52 | arguments = commandArgs(TRUE)) { 53 | 54 | # warn if the user has supplied a 'file' argument but no such file exists 55 | if (!missing(file) && !file.exists(file)) { 56 | warning(sprintf("configuration file '%s' does not exist", file)) 57 | file <- NULL # prevent errors/warnings downstream 58 | } 59 | 60 | # get flags file from training_run() if available and not specified 61 | if (missing(file) && !is.null(.globals$run_dir$flags_file)) { 62 | if (file.exists(.globals$run_dir$flags_file)) { 63 | file <- .globals$run_dir$flags_file 64 | } else { 65 | warning(sprintf("configuration file '%s' does not exist", file)) 66 | file <- NULL # prevent errors/warnings downstream 67 | } 68 | } 69 | 70 | # get config from training_run() if available and it's not specified 71 | if (missing(config) && !is.null(.globals$run_dir$config)) 72 | config <- .globals$run_dir$config 73 | 74 | 75 | # create an empty FLAGS structure 76 | FLAGS <- structure(class = "tfruns_flags", 77 | types = character(), 78 | defaults = list(), 79 | docstrings = character(), 80 | list() 81 | ) 82 | 83 | # resolve flag definitions (accept a list) 84 | flag_defs <- list(...) 85 | if (length(flag_defs) == 1 && !inherits(flag_defs[[1]], "tfruns_flag")) 86 | flag_defs <- flag_defs[[1]] 87 | 88 | # add the flags 89 | for (flag in flag_defs) 90 | FLAGS <- add_flag(FLAGS, flag$name, flag$type, flag$default, flag$description) 91 | 92 | # check for --help 93 | if ("--help" %in% arguments) 94 | print_flags_and_quit(FLAGS) 95 | 96 | # parse the command line / config file 97 | if (length(flag_defs) > 0) 98 | FLAGS <- parse_flags(FLAGS, config, file, arguments) 99 | 100 | # write flags 101 | write_run_metadata("flags", FLAGS) 102 | 103 | # return flags 104 | FLAGS 105 | } 106 | 107 | #' @rdname flags 108 | #' @export 109 | flag_numeric <- function(name, default, description = NULL) { 110 | flag_definition( 111 | name = name, 112 | type = "numeric", 113 | default = as.double(default), 114 | description = description 115 | ) 116 | } 117 | 118 | #' @rdname flags 119 | #' @export 120 | flag_integer <- function(name, default, description = NULL) { 121 | flag_definition( 122 | name = name, 123 | type = "integer", 124 | default = as.integer(default), 125 | description = description 126 | ) 127 | } 128 | 129 | #' @rdname flags 130 | #' @export 131 | flag_boolean <- function(name, default, description = NULL) { 132 | flag_definition( 133 | name = name, 134 | type = "boolean", 135 | default = as.logical(default), 136 | description = description 137 | ) 138 | } 139 | 140 | #' @rdname flags 141 | #' @export 142 | flag_string <- function(name, default, description = NULL) { 143 | flag_definition( 144 | name = name, 145 | type = "string", 146 | default = as.character(default), 147 | description = description 148 | ) 149 | } 150 | 151 | flag_definition <- function(name, type, default, description) { 152 | structure(class = "tfruns_flag", list( 153 | name = name, 154 | type = type, 155 | default = default, 156 | description = description 157 | )) 158 | } 159 | 160 | 161 | parse_flags <- function(FLAGS, config, file, arguments) { 162 | 163 | # read configuration file if it exists 164 | if (!is.null(file) && file.exists(file)) { 165 | 166 | # first load the raw YAML so we can provide defaults from FLAGS 167 | # (this obviates the need to also provide the defaults in flags.yml) 168 | config_yaml <- yaml::yaml.load_file(file) 169 | 170 | # check to see whether this is a profile-oriented config file 171 | 172 | # see what the structure of the config file is. if it's all lists at the 173 | # top level then treat it as a traditional profile-oriented config file 174 | if (all(vapply(config_yaml, is.list, TRUE))) { 175 | # synthesize default section based on what's already in the config file 176 | # and the defaults provided inline 177 | config_yaml[["default"]] <- config::merge(as.vector(FLAGS), 178 | config_yaml[["default"]]) 179 | } else { 180 | # synthesize default section from the values at the top level 181 | config_yaml[["default"]] <- config::merge(as.vector(FLAGS), config_yaml) 182 | } 183 | 184 | # now write this to a temp file which we will read with config::get 185 | flags_tmp <- tempfile(pattern = "tfruns-flags", fileext = ".yml") 186 | on.exit(unlink(flags_tmp), add = TRUE) 187 | writeLines(yaml::as.yaml(config_yaml), flags_tmp) 188 | 189 | # read the config 190 | flags <- config::get(config = config, file = flags_tmp, use_parent = FALSE) 191 | 192 | } else { 193 | 194 | # no config file, just use default FLAGS 195 | flags <- as.vector(FLAGS) 196 | 197 | } 198 | 199 | # merge with command line arguments (if any) 200 | flags <- config::merge(flags, parse_command_line(arguments)) 201 | 202 | # merge with flags provided via training_run() 203 | tr_flags <- .globals$run_dir$flags 204 | tr_flag_names <- names(tr_flags) 205 | if (!is.null(tr_flags) && !is.null(tr_flag_names)) { 206 | tr_flags <- as.character(tr_flags) 207 | tr_flags_arguments <- character() 208 | for (i in 1:length(tr_flags)) { 209 | if (nzchar(tr_flag_names[[i]])) { 210 | name <- paste0("--", tr_flag_names[[i]]) 211 | value <- tr_flags[[i]] 212 | } 213 | tr_flags_arguments <- c(tr_flags_arguments, name, value) 214 | } 215 | flags <- config::merge(flags, parse_command_line(tr_flags_arguments)) 216 | } 217 | 218 | # discover undeclared arguments 219 | undeclared <- setdiff(names(flags), names(FLAGS)) 220 | if (length(undeclared) > 0) { 221 | stop("The following flags were provided but not declared: ", 222 | paste(undeclared, collapse = ", "), call. = FALSE) 223 | } 224 | 225 | # set discovered values in FLAGS (set one at a time so that 226 | # we can do type coercion and preserve the attributes of FLAGS) 227 | names <- names(FLAGS) 228 | types <- attr(FLAGS, "types") 229 | for (i in 1:length(FLAGS)) { 230 | name <- names[[i]] 231 | type <- types[[i]] 232 | value <- suppressWarnings(switch(type, 233 | numeric = as.double(flags[[name]]), 234 | integer = as.integer(flags[[name]]), 235 | boolean = as.logical(flags[[name]]), 236 | string = as.character(flags[[name]]) 237 | )) 238 | # error if length > 1 (no support for parsing lists @ the command line) 239 | if (length(value) > 1) { 240 | stop('flag "', name, '" has length > 1 (only scalar flag values are supported)') 241 | } 242 | # error if type coersion fails 243 | if (is.na(value)) { 244 | value <- paste0("'", flags[[name]], "'") 245 | stop("Unable to convert flag '", name, "' with value ", value, 246 | " to type ", type, call. = FALSE) 247 | } 248 | FLAGS[[name]] <- value 249 | } 250 | 251 | # return flags 252 | FLAGS 253 | } 254 | 255 | #' @export 256 | as.data.frame.tfruns_flags <- function(x, ...) { 257 | if (length(x) > 0) { 258 | data.frame(stringsAsFactors = FALSE, 259 | name = names(x), 260 | type = attr(x, "types"), 261 | value = as.character(x), 262 | description = attr(x, "descriptions") 263 | ) 264 | } else { 265 | data.frame(stringsAsFactors = FALSE, 266 | name = character(), 267 | type = character(), 268 | value = character(), 269 | description = character()) 270 | } 271 | } 272 | 273 | #' @export 274 | print.tfruns_flags <- function(x, ...) { 275 | print(as.data.frame(x), row.names = FALSE) 276 | } 277 | 278 | # add a flag definition 279 | add_flag <- function(FLAGS, name, type, default, description) { 280 | 281 | # validate uniqueness of name 282 | if (!is.null(FLAGS[[name]])) 283 | stop("A flag named '", name, "' has already been defined", call. = FALSE) 284 | 285 | # add flag 286 | FLAGS[[name]] <- default 287 | attr(FLAGS, "types") <- c(attr(FLAGS, "types"), type) 288 | attr(FLAGS, "defaults") <- c(attr(FLAGS, "defaults"), default) 289 | attr(FLAGS, "descriptions") <- c(attr(FLAGS, "descriptions"), 290 | ifelse(is.null(description), NA, description)) 291 | 292 | # return flags 293 | FLAGS 294 | } 295 | 296 | # parse command line arguments 297 | parse_command_line <- function(arguments) { 298 | 299 | # initialize some state 300 | values <- list() 301 | 302 | i <- 0; n <- length(arguments) 303 | while (i < n) { 304 | i <- i + 1 305 | argument <- arguments[[i]] 306 | 307 | # skip any command line arguments without a '--' prefix 308 | if (!grepl("^--", argument)) 309 | next 310 | 311 | # terminate if we see "--args" (implies passthrough args) 312 | if (grepl("^--args$", argument)) 313 | break 314 | 315 | # check to see if an '=' was specified for this argument 316 | equals_idx <- regexpr("=", argument) 317 | if (identical(c(equals_idx), -1L)) { 318 | # no '='; the next argument is the value for this key 319 | key <- substring(argument, 3) 320 | val <- arguments[[i + 1]] 321 | i <- i + 1 322 | } else { 323 | # found a '='; the next argument is all the text following 324 | # that character 325 | key <- substring(argument, 3, equals_idx - 1) 326 | val <- substring(argument, equals_idx + 1) 327 | } 328 | 329 | # convert '-' to '_' in key 330 | key <- gsub("-", "_", key) 331 | 332 | # update our map of argument values 333 | values[[key]] <- yaml::yaml.load(val) 334 | } 335 | 336 | values 337 | 338 | } 339 | 340 | print_flags_and_quit <- function(FLAGS) { 341 | flags_df <- as.data.frame(FLAGS) 342 | if (nrow(flags_df) > 0) { 343 | max_name_len <- max(nchar(flags_df$name)) 344 | cat("\nOptions:\n") 345 | for (i in 1:nrow(flags_df)) { 346 | row <- flags_df[i,] 347 | flag <- paste0(" --", gsub("_", "-", row$name)) 348 | flag <- format(flag, width = max_name_len + 4) 349 | description <- row$description 350 | if (is.na(description)) 351 | description <- "" 352 | else 353 | description <- paste0(description, " ") 354 | description <- paste0(description, "(", row$value, ")") 355 | cat(sprintf("%s %s\n", flag, description)) 356 | } 357 | cat("\n") 358 | } 359 | 360 | fn_quit <- utils::getFromNamespace("quit", asNamespace("base")) 361 | fn_quit(save = "no") 362 | } 363 | 364 | 365 | 366 | 367 | 368 | 369 | -------------------------------------------------------------------------------- /R/ls_runs.R: -------------------------------------------------------------------------------- 1 | 2 | #' List or view training runs 3 | #' 4 | #' @param subset Logical expression indicating rows to keep (missing values are 5 | #' taken as false). See [subset()]. 6 | #' @param order Columns to order by (defaults to run start time) 7 | #' @param decreasing `TRUE` to use decreasing order (e.g. list most recent runs 8 | #' first) 9 | #' @param latest_n Limit query to the `latest_n` most recent runs 10 | #' @param runs_dir Directory containing runs. Defaults to "runs" beneath the 11 | #' current working directory (or to the value of the `tfruns.runs_dir` R 12 | #' option if specified). 13 | #' 14 | #' @return Data frame with training runs 15 | #' 16 | #' @details When printing the results of `ls_runs()`, only `run_dir`, 17 | #' `metric_loss`, `metric_val_loss`, and any columns specified in `order` will 18 | #' be printed. 19 | #' 20 | #' To view all fields, use `View(ls_runs())`. 21 | #' 22 | #' @export 23 | ls_runs <- function(subset = NULL, 24 | order = "start", 25 | decreasing = TRUE, 26 | latest_n = NULL, 27 | runs_dir = getOption("tfruns.runs_dir", "runs")) { 28 | 29 | # default empty run list 30 | run_list <- NULL 31 | 32 | if (utils::file_test("-d", runs_dir)) { 33 | 34 | # list runs 35 | runs <- list_run_dirs(latest_n = latest_n, runs_dir = runs_dir) 36 | 37 | # popuate data frame from runs 38 | for (run in runs) { 39 | run_df <- run_record(run) 40 | if (is.null(run_list)) 41 | run_list <- run_df 42 | else 43 | run_list <- combine_runs(run_list, run_df) 44 | } 45 | } 46 | 47 | if (!is.null(run_list)) { 48 | 49 | # resolve order 50 | order <- tidyselect::vars_select(colnames(run_list), !! rlang::enquo(order)) 51 | 52 | # build order expression 53 | order_cols <- paste(paste0("run_list$", order), collapse = ",") 54 | order_expr <- sprintf("run_list[order(%s, decreasing = %s),]", 55 | order_cols, deparse(decreasing)) 56 | run_list <- eval(parse(text = order_expr)) 57 | 58 | # convert date columns 59 | as_date <- function(value) { 60 | if (!is.null(value)) 61 | as.POSIXct(value, tz = "GMT", origin = "1970-01-01") 62 | else 63 | NULL 64 | } 65 | run_list$start <- as_date(run_list$start) 66 | run_list$end <- as_date(run_list$end) 67 | run_list$cloudml_created <- as_date(run_list[["cloudml_created"]]) 68 | run_list$cloudml_start <- as_date(run_list[["cloudml_start"]]) 69 | run_list$cloudml_end <- as_date(run_list[["cloudml_end"]]) 70 | 71 | } else { 72 | run_list <- data.frame( 73 | type = character(), 74 | run_dir = character(), 75 | start = numeric(), 76 | end = numeric(), 77 | stringsAsFactors = FALSE 78 | ) 79 | } 80 | 81 | # apply subset 82 | if (!missing(subset)) { 83 | subset_call <- substitute(subset) 84 | rows <- eval(subset_call, run_list) 85 | run_list <- run_list[rows, ] 86 | rownames(run_list) <- seq(length = nrow(run_list)) 87 | } 88 | 89 | # return runs 90 | return_runs(run_list, order) 91 | } 92 | 93 | 94 | 95 | #' Latest training run 96 | #' 97 | #' @inheritParams ls_runs 98 | #' 99 | #' @return Named list with run attributes (or `NULL` if no runs found) 100 | #' 101 | #' @export 102 | latest_run <- function(runs_dir = getOption("tfruns.runs_dir", "runs")) { 103 | latest_run_df <- ls_runs(latest_n = 1, runs_dir = runs_dir) 104 | if (nrow(latest_run_df) > 0) { 105 | as_run_info(latest_run_df) 106 | } else { 107 | NULL 108 | } 109 | } 110 | 111 | 112 | #' Summary of training run 113 | #' 114 | #' @param run_dir Training run directory or data frame returned from 115 | #' [ls_runs()]. 116 | #' 117 | #' @return Training run summary object with timing, flags, model info, training 118 | #' and evaluation metrics, etc. If more than one `run_dir` is passed then 119 | #' a list of training run summary objects is returned. 120 | #' 121 | #' @seealso [view_run()] 122 | #' 123 | #' @export 124 | run_info <- function(run_dir) { 125 | run_dir <- as_run_dir(run_dir) 126 | if (length(run_dir) == 0) 127 | list() 128 | else if (length(run_dir) == 1) 129 | as_run_info(run_record(run_dir)) 130 | else 131 | lapply(run_dir, function(dir) { 132 | as_run_info(run_record(dir)) 133 | }) 134 | } 135 | 136 | 137 | #' @importFrom utils str 138 | #' 139 | #' @export 140 | print.tfruns_run <- function(x, ...) { 141 | 142 | # summarize longer fields 143 | summarize <- function(field, summary) { 144 | if (!is.null(x[[field]])) 145 | summary 146 | else 147 | NULL 148 | } 149 | x$metrics <- summarize("metrics", "(metrics data frame)") 150 | x$model <- summarize("model", "(model summary)") 151 | x$source_code <- summarize("source_code", "(source archive)") 152 | x$output <- summarize("output", "(script ouptut)") 153 | 154 | # print 155 | str(x, no.list = TRUE) 156 | } 157 | 158 | #' @export 159 | print.tfruns_model_summary <- function(x, ...) { 160 | cat(x) 161 | } 162 | 163 | #' @export 164 | print.tfruns_runs_df <- function(x, ...) { 165 | cols <- colnames(x) 166 | if (nrow(x) == 0) { 167 | cat("No training runs found.\n") 168 | } else if (nrow(x) == 1) { 169 | print(as_run_info(x)) 170 | } else { 171 | # if no subsetting of columns has taken place then use default display 172 | if (identical(colnames(x), attr(x, "original_cols"))) { 173 | 174 | # calculate and apply display columns 175 | order_cols <- attr(x, "order") 176 | eval_cols <- cols[grepl("^eval_", cols)] 177 | metric_cols <- cols[grepl("^metric_", cols)] 178 | display_cols <- unique(c("run_dir", order_cols, eval_cols, metric_cols)) 179 | display_cols <- intersect(display_cols, cols) 180 | x <- x[, display_cols, drop = FALSE] 181 | 182 | # print extra cols 183 | extra_cols <- cols[!is.element(cols, display_cols)] 184 | if (length(extra_cols) > 0) { 185 | extra_output <- paste0(extra_cols, collapse = ", ") 186 | extra_output <- paste( 187 | paste("# ", strwrap(extra_output)), 188 | collapse = "\n" 189 | ) 190 | extra_output <- paste0( 191 | "# ... with ", length(extra_cols), " more columns:\n", 192 | extra_output 193 | ) 194 | } 195 | } else { 196 | extra_output <- NULL 197 | } 198 | 199 | # see if there are extra rows 200 | original_rows <- nrow(x) 201 | x <- utils::head(x, n = 10) 202 | extra_rows <- original_rows - nrow(x) 203 | if (extra_rows > 0) { 204 | extra_output <- paste0("# ... with ", extra_rows, " more rows\n", 205 | extra_output) 206 | } 207 | 208 | # print header 209 | cat(sprintf("Data frame: %d x %d", original_rows, length(cols)), "\n") 210 | 211 | # print with default df method 212 | cls <- class(x) 213 | cls <- cls[cls != "tfruns_runs_df"] 214 | class(x) <- cls 215 | print(x) 216 | 217 | # print extra output 218 | if (!is.null(extra_output)) 219 | cat(extra_output) 220 | } 221 | } 222 | 223 | 224 | list_run_dirs <- function(latest_n = NULL, runs_dir) { 225 | 226 | # return empty character vector for dir not found 227 | if (!utils::file_test("-d", runs_dir)) 228 | return(character()) 229 | 230 | # list directories 231 | runs <- list.files(runs_dir, full.names = FALSE) 232 | runs <- runs[utils::file_test("-d", file.path(runs_dir, runs, "tfruns.d"))] 233 | 234 | # filter and order 235 | if (length(runs) > 0) { 236 | runs <- runs[order(runs, decreasing = TRUE)] 237 | if (!is.null(latest_n)) 238 | runs <- runs[1:min(length(runs),latest_n)] 239 | } 240 | 241 | # return runs 242 | file.path(runs_dir, runs) 243 | } 244 | 245 | run_record <- function(run_dir) { 246 | 247 | # validate that it exists 248 | if (!utils::file_test("-d", run_dir)) 249 | stop("Run directory ", run_dir, " does not exist", call. = FALSE) 250 | 251 | # compute run name and meta dir 252 | run <- basename(run_dir) 253 | meta_dir <- file.path(run_dir, "tfruns.d") 254 | props_dir <- file.path(meta_dir, "properties") 255 | if (!utils::file_test("-d", props_dir)) 256 | props_dir <- NULL 257 | 258 | # read all properties into a list 259 | read_properties <- function() { 260 | if (!is.null(props_dir) && file.exists(props_dir)) { 261 | properties <- list.files(props_dir) 262 | values <- lapply(properties, function(file) { 263 | paste(readLines(file.path(props_dir, file)), collapse = "\n") 264 | }) 265 | names(values) <- properties 266 | 267 | # default 'type' and 'context' (data migration) 268 | if (is.null(values$type) || identical(values$type, 'local')) 269 | values$type <- 'training' 270 | if (is.null(values$context)) 271 | values$context <- 'local' 272 | 273 | # return values 274 | values 275 | } else { 276 | list() 277 | } 278 | } 279 | 280 | # type converters for properties 281 | as_type <- function(properties, name, converter) { 282 | value <- properties[[name]] 283 | if (is.null(value)) 284 | NULL 285 | else 286 | converter(value) 287 | } 288 | as_numeric <- function(properties, name) { 289 | as_type(properties, name, as.numeric) 290 | } 291 | as_integer <- function(properties, name) { 292 | as_type(properties, name, as.integer) 293 | } 294 | as_logical <- function(properties, name) { 295 | as_type(properties, name, function(value) { 296 | if (value %in% c("TRUE", "true", "yes", "1")) 297 | value <- TRUE 298 | else if (value %in% c("FALSE", "false", "no", "0")) 299 | value <- FALSE 300 | as.logical(value) 301 | }) 302 | } 303 | 304 | # function to read columns from a json file 305 | read_json_columns <- function(file, prefix) { 306 | json_path <- file.path(meta_dir, file) 307 | if (file.exists(json_path)) { 308 | columns <- jsonlite::read_json(json_path) 309 | if (length(columns) > 0) { 310 | names(columns) <- paste0(prefix, "_", names(columns)) 311 | } 312 | columns 313 | } else { 314 | NULL 315 | } 316 | } 317 | 318 | # core columns 319 | columns <- list() 320 | columns$run_dir <- run_dir 321 | 322 | # read properties and do type conversions for known values 323 | properties <- read_properties() 324 | properties$start <- as_numeric(properties, "start") 325 | properties$end <- as_numeric(properties, "end") 326 | properties$samples <- as_integer(properties, "samples") 327 | properties$validation_samples <- as_integer(properties, "validation_samples") 328 | for (unit in valid_steps_units) 329 | properties[[unit]] <- as_integer(properties, unit) 330 | properties$batch_size <- as_integer(properties, "batch_size") 331 | properties$completed <- as_logical(properties, "completed") 332 | properties$learning_rate <- as_numeric(properties, "learning_rate") 333 | properties$cloudml_created <- as_integer(properties, "cloudml_created") 334 | properties$cloudml_start <- as_integer(properties, "cloudml_start") 335 | properties$cloudml_end <- as_integer(properties, "cloudml_end") 336 | properties$cloudml_ml_units <- as_numeric(properties, "cloudml_ml_units") 337 | 338 | # add properties to columns 339 | columns <- append(columns, properties) 340 | 341 | # evaluation 342 | columns <- append(columns, read_json_columns("evaluation.json", "eval")) 343 | 344 | # metrics 345 | epochs_completed <- 0L 346 | metrics_json_path <- file.path(meta_dir, "metrics.json") 347 | if (file.exists(metrics_json_path)) { 348 | # read metrics 349 | metrics <- jsonlite::read_json(metrics_json_path, simplifyVector = TRUE) 350 | if (length(metrics) > 0) { 351 | for (metric in names(metrics)) { 352 | if (metric == "epoch") 353 | next 354 | values <- metrics[[metric]] 355 | available_values <- values[!is.na(values)] 356 | epochs_completed <- length(available_values) 357 | if (epochs_completed > 0) { 358 | last_value <- available_values[[epochs_completed]] 359 | columns[[paste0("metric_", metric)]] <- last_value 360 | } 361 | } 362 | } 363 | } 364 | 365 | steps_completed_unit <- get_steps_completed_unit(get_steps_unit(columns)) 366 | # epochs completed 367 | columns[[steps_completed_unit]] <- epochs_completed 368 | 369 | # flags 370 | columns <- append(columns, read_json_columns("flags.json", "flag")) 371 | 372 | # error 373 | error_json_path <- file.path(meta_dir, "error.json") 374 | if (file.exists(error_json_path)) { 375 | error <- jsonlite::read_json(error_json_path, simplifyVector = TRUE) 376 | columns[["error_message"]] <- error$message 377 | columns[["error_traceback"]] <- paste(error$traceback, collapse = "\n") 378 | } 379 | 380 | 381 | # add metrics and source fields 382 | meta_dir <- meta_dir(run_dir, create = FALSE) 383 | metrics_json <- file.path(meta_dir, "metrics.json") 384 | if (file.exists(metrics_json)) 385 | columns$metrics <- metrics_json 386 | source_code <- file.path(meta_dir, "source.tar.gz") 387 | if (file.exists(source_code)) 388 | columns$source_code <- source_code 389 | 390 | # convert to data frame for calls to rbind 391 | as.data.frame(columns, stringsAsFactors = FALSE) 392 | } 393 | 394 | combine_runs <- function(x, y) { 395 | x[, c(as.character(setdiff(colnames(y), colnames(x))))] <- NA 396 | y[, c(as.character(setdiff(colnames(x), colnames(y))))] <- NA 397 | rbind(x, y) 398 | } 399 | 400 | return_runs <- function(runs, order = NULL) { 401 | 402 | # re-order columns 403 | select_cols <- function(cols) { 404 | intersect(cols, colnames(runs)) 405 | } 406 | cols_with_prefix <- function(prefix) { 407 | cols <- colnames(runs) 408 | cols[grepl(paste0("^", prefix, "_"), cols)] 409 | } 410 | cols <- character() 411 | cols <- c(cols, cols_with_prefix("eval")) 412 | cols <- c(cols, cols_with_prefix("metric")) 413 | cols <- c(cols, cols_with_prefix("flag")) 414 | cols <- c(cols, select_cols(c("samples", "validation_samples"))) 415 | cols <- c(cols, select_cols(c("batch_size"))) 416 | for (unit in valid_steps_units) 417 | cols <- c(cols, select_cols(c(unit, paste0(unit, "_completed")))) 418 | cols <- c(cols, select_cols(c("metrics"))) 419 | cols <- c(cols, select_cols(c("model", "loss_function", "optimizer", "learning_rate"))) 420 | cols <- c(cols, select_cols(c("script", "source"))) 421 | cols <- c(cols, select_cols(c("start", "end", "completed"))) 422 | cols <- c(cols, select_cols(c("output", "error_message", "error_traceback"))) 423 | cols <- c(cols, select_cols(c("source_code"))) 424 | cols <- c(cols, select_cols(c("context", "type"))) 425 | cols <- c(cols, setdiff(colnames(runs), cols)) 426 | 427 | # promote any ordered columns to the front 428 | if (identical(unname(order), "start")) 429 | order <- NULL 430 | initial_cols <- c(select_cols(c("run_dir")), order) 431 | cols <- setdiff(cols, initial_cols) 432 | cols <- c(initial_cols, cols) 433 | 434 | # re-order cols (always have type and run_dir at the beginning) 435 | runs <- runs[, cols] 436 | 437 | # apply special class and add order attribute 438 | class(runs) <- c("tfruns_runs_df", class(runs)) 439 | attr(runs, "order") <- order 440 | attr(runs, "original_cols") <- colnames(runs) 441 | 442 | # return runs 443 | runs 444 | } 445 | 446 | as_run_info <- function(run_record) { 447 | 448 | # re-order columns 449 | runs <- return_runs(run_record) 450 | 451 | # convert to list 452 | run_info <- list() 453 | for (col in colnames(runs)) 454 | run_info[[col]] <- runs[[col]] 455 | 456 | # give the model a class that will make it print nicely 457 | if (!is.null(run_info$model)) 458 | class(run_info$model) <- c("tfruns_model_summary", "character") 459 | 460 | # convert metrics to data frame 461 | if (!is.null(run_info$metrics) && file.exists(run_info$metrics)) { 462 | run_info$metrics <- as.data.frame( 463 | jsonlite::read_json(run_info$metrics, simplifyVector = TRUE)) 464 | } 465 | 466 | # return with class 467 | structure(class = "tfruns_run", run_info) 468 | } 469 | 470 | run_source_code <- function(script, run_dir) { 471 | source_tarball <- file.path(meta_dir(run_dir), "source.tar.gz") 472 | if (file.exists(source_tarball)) { 473 | source_tmp_dir <- tempfile() 474 | on.exit(unlink(source_tmp_dir, recursive = TRUE)) 475 | if (is_windows()) 476 | tar <- "internal" 477 | else 478 | tar = Sys.getenv("tar") 479 | utils::untar(source_tarball, exdir = source_tmp_dir, compressed = TRUE, tar = tar) 480 | source_dir <- file.path(source_tmp_dir, "source") 481 | source_files <- list.files(source_dir, recursive = TRUE) 482 | source_files <- c(script, source_files[!grepl(paste0("^", script, "$"), source_files)]) 483 | names(source_files) <- source_files 484 | lapply(source_files, function(file) { 485 | paste(readLines(file.path(source_dir, file)), collapse = "\n") 486 | }) 487 | } else { 488 | list() 489 | } 490 | } 491 | 492 | valid_steps_units <- c("steps", "epochs") 493 | 494 | get_steps_unit <- function(run) { 495 | steps_unit <- "steps" 496 | for (unit in valid_steps_units) { 497 | if (!is.null(run[[unit]])) { 498 | steps_unit <- unit 499 | break 500 | } 501 | } 502 | steps_unit 503 | } 504 | 505 | get_steps_completed_unit <- function(steps_unit) { 506 | paste0(steps_unit, "_completed") 507 | } 508 | 509 | -------------------------------------------------------------------------------- /R/metrics_viewer.R: -------------------------------------------------------------------------------- 1 | 2 | #' View metrics for a training run 3 | #' 4 | #' Interactive D3 visualization of metrics for a training run. Metrics will 5 | #' be displayed in the RStudio Viewer (if available), otherwise will be 6 | #' displayed in an external web browser. 7 | #' 8 | #' @param metrics Data frame containing run metrics 9 | #' @param viewer Viewer object returned from `view_run_metrics()`. 10 | #' 11 | #' @template roxlate-metrics-format 12 | #' 13 | #' @note 14 | #' Metrics named `"acc"` or `"accuracy"` will automatically use `1.0` as the 15 | #' maximum value on their y-axis scale. 16 | #' 17 | #' @section Realtime Updates: 18 | #' 19 | #' Metrics can be updated in real-time by calling the `update_run_metrics()` 20 | #' with the run viewer instance returned from `view_run_metrics()`. For example: 21 | #' 22 | #' ``` 23 | #' # view metrics 24 | #' viewer <- view_run_metrics(metrics) 25 | #' 26 | #' # update with new metrics 27 | #' update_run_metrics(viewer, updated_metrics) 28 | #' ``` 29 | #' 30 | #' @seealso write_run_metrics 31 | #' @export 32 | view_run_metrics <- function(metrics) { 33 | 34 | # create a new temp directory for the viewer's UI/data 35 | viewer_dir <- tempfile("tfruns-metrics") 36 | dir.create(viewer_dir, recursive = TRUE) 37 | 38 | # create the metrics_viewer instance 39 | metrics_viewer <- structure(class = "tfruns_metrics_viewer", list( 40 | viewer_dir = viewer_dir 41 | )) 42 | 43 | # write the history 44 | update_run_metrics(metrics_viewer, metrics) 45 | 46 | # view it 47 | viewer <- getOption("viewer", default = browser_viewer(viewer_dir)) 48 | viewer(file.path(viewer_dir, "index.html")) 49 | 50 | # return metrics_viewer instance (invisibly) for subsequent 51 | # calls to update_run_history 52 | invisible(metrics_viewer) 53 | } 54 | 55 | 56 | #' @rdname view_run_metrics 57 | #' @export 58 | update_run_metrics <- function(viewer, metrics) { 59 | 60 | # re-write index.html with embedded metrics 61 | metrics_json <- jsonlite::toJSON(metrics, dataframe = "columns", na = "null") 62 | metrics_html_path <- file.path(viewer$viewer_dir, "index.html") 63 | render_view("metrics",metrics_html_path, variables = list(metrics_json = metrics_json)) 64 | 65 | # write metrics.json for polling 66 | metrics_json_path <- file.path(viewer$viewer_dir, "metrics.json") 67 | write_metrics_json(metrics, metrics_json_path) 68 | } 69 | 70 | # write metrics as json 71 | write_metrics_json <- function(metrics, path) { 72 | jsonlite::write_json(metrics, path, 73 | pretty = TRUE, 74 | dataframe = "columns", 75 | na = "null") 76 | } 77 | 78 | # non-rstudio viewer function 79 | browser_viewer <- function(viewer_dir, browser = utils::browseURL) { 80 | 81 | function(url) { 82 | # determine help server port 83 | port <- tools::startDynamicHelp(NA) 84 | if (port <= 0) 85 | port <- tools::startDynamicHelp(TRUE) 86 | if (port <= 0) { 87 | warning("Unable to view run metrics (couldn't access help server port)", 88 | call. = FALSE) 89 | return(invisible(NULL)) 90 | } 91 | 92 | # determine path to history html 93 | path <- paste("/session", basename(viewer_dir), basename(url), sep = "/") 94 | 95 | # build URL and browse it 96 | url <- paste0("http://127.0.0.1:", port, path) 97 | browser(url) 98 | } 99 | } 100 | 101 | 102 | -------------------------------------------------------------------------------- /R/package.R: -------------------------------------------------------------------------------- 1 | 2 | 3 | .globals <- new.env(parent = emptyenv()) 4 | 5 | # run dir 6 | .globals$run_dir <- new.env(parent = emptyenv()) 7 | .globals$run_dir$path <- NULL 8 | .globals$run_dir$flags <- NULL 9 | .globals$run_dir$flags_file <- NULL 10 | .globals$run_dir$pending_writes <- new.env(parent = emptyenv()) 11 | 12 | # view components 13 | .globals$view_components <- new.env(parent = emptyenv()) 14 | -------------------------------------------------------------------------------- /R/reexports.R: -------------------------------------------------------------------------------- 1 | 2 | 3 | #' @keywords internal 4 | #' @importFrom magrittr %>% 5 | #' @export 6 | magrittr::`%>%` 7 | 8 | 9 | -------------------------------------------------------------------------------- /R/run_dir.R: -------------------------------------------------------------------------------- 1 | 2 | #' Current run directory 3 | #' 4 | #' Returns the current training run directory. If a training run is 5 | #' not currently active (see [is_run_active()]) then the current 6 | #' working directory is returned. 7 | #' 8 | #' @return Active run direcotry (or current working directory as a fallback) 9 | #' 10 | #' @export 11 | run_dir <- function() { 12 | 13 | # do we already have a run_dir? 14 | if (is_run_active()) { 15 | 16 | .globals$run_dir$path 17 | 18 | # no run_dir currently established 19 | } else { 20 | 21 | getwd() 22 | 23 | } 24 | } 25 | 26 | 27 | #' Check for an active training run 28 | #' 29 | #' @return `TRUE` if a training tun is currently active 30 | #' 31 | #' @export 32 | is_run_active <- function() { 33 | !is.null(.globals$run_dir$path) 34 | } 35 | 36 | 37 | #' Create a unique run directory 38 | #' 39 | #' Create a new uniquely named run directory within the specified `runs_dir`. 40 | #' 41 | #' The directory name will be a timestamp (in GMT time). If a duplicate name is 42 | #' generated then the function will wait long enough to return a unique one. 43 | #' 44 | #' @inheritParams ls_runs 45 | #' @param seconds_scale Decimal scale for the seconds component of the 46 | #' timestamp. Defaults to 0 which results in only the rounded seconds value 47 | #' being used in the timestamp. Specify larger numbers to include a decimal 48 | #' component (useful if you need to create many unique run directories at the 49 | #' same time). 50 | #' 51 | #' @export 52 | unique_run_dir <- function(runs_dir = getOption("tfruns.runs_dir", "runs"), 53 | seconds_scale = 0) { 54 | 55 | # determine seconds format 56 | if (seconds_scale == 0) 57 | seconds_format <- "S" 58 | else 59 | seconds_format <- paste0("OS", seconds_scale) 60 | 61 | # loop while waiting to create a unique run directory 62 | while(TRUE) { 63 | run_dir <- file.path( 64 | runs_dir, 65 | paste0(strftime( 66 | Sys.time(), 67 | format = paste0("%Y-%m-%dT%H-%M-%", seconds_format, "Z"), 68 | tz = "GMT") 69 | ) 70 | ) 71 | if (!file.exists(run_dir)) { 72 | dir.create(run_dir, recursive = TRUE) 73 | return(run_dir) 74 | } 75 | # sleep for an appropriate interval before trying again 76 | else { 77 | sleep_for <- (1 / 10^seconds_scale) * 0.5 78 | Sys.sleep(sleep_for) 79 | } 80 | 81 | } 82 | } 83 | 84 | 85 | #' Extract run directory from an object 86 | #' 87 | #' @param x Object to extract run directory from 88 | #' 89 | #' @return Run directory path(s) 90 | #' 91 | #' @keywords internal 92 | #' 93 | #' @export 94 | as_run_dir <- function(x) { 95 | UseMethod("as_run_dir") 96 | } 97 | 98 | #' @export 99 | as_run_dir.character <- function(x) { 100 | if (all(utils::file_test("-d", x))) 101 | x 102 | else { 103 | runs_dir <- getOption("tfruns.runs_dir", "runs") 104 | run_dir <- file.path(runs_dir, x) 105 | if (all(utils::file_test("-d", run_dir))) 106 | run_dir 107 | else 108 | x 109 | } 110 | } 111 | 112 | #' @export 113 | as_run_dir.tfruns_run <- function(x) { 114 | x$run_dir 115 | } 116 | 117 | #' @export 118 | as_run_dir.list <- function(x) { 119 | if (!is.null(x$run_dir)) 120 | x$run_dir 121 | else 122 | stop("List does not contain a run_dir") 123 | } 124 | 125 | #' @export 126 | as_run_dir.data.frame <- function(x) { 127 | if (!is.null(x$run_dir)) 128 | x$run_dir 129 | else 130 | stop("Data frame does not contain a run_dir") 131 | } 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | -------------------------------------------------------------------------------- /R/run_metadata.R: -------------------------------------------------------------------------------- 1 | 2 | 3 | #' Write run metadata 4 | #' 5 | #' Record various types of training run metadata This function can be called 6 | #' even when a run directory isn't active (metadata will only be written if 7 | #' and when a run directory is initialized). 8 | #' 9 | #' @param type Type of metadata to write. Standard types include "flags", 10 | #' "sources", "properties", "metrics", and "evaluation". You can 11 | #' also specify a custom type (see *Custom Types* section below). 12 | #' 13 | #' @param data Metadata to write: 14 | #' 15 | #' - "flags" --- Named list of training flags 16 | #' - "source" --- Directory to copy source files from 17 | #' - "properties" --- Named list of arbitrary properties. Note 18 | #' that properties will be stored as strings. 19 | #' - "metrics" --- Data frame with training run metrics 20 | #' (see *Metrics Data Frame* below). 21 | #' - "evaluation" --- Named list of evaluation metrics. 22 | #' - "error" --- Named list with 'message' and 'traceback' 23 | #' - "\" -- Function used to write the data 24 | #' (see *Custom Types* section below). 25 | #' 26 | #' @param run_dir Run directory to write metadata into (defaults 27 | #' to currently active run) 28 | #' 29 | #' @template roxlate-metrics-format 30 | #' 31 | #' @section Custom Types: 32 | #' 33 | #' You can pass a type with an arbitary name along with a function that 34 | #' should be used to writes the data. The function will be passed a 35 | #' single `data_dir` argument. For example: 36 | #' 37 | #' ```r 38 | #' write_run_metadata("images", function(data_dir) { 39 | #' # write into data_dir here 40 | #' }) 41 | #' ```` 42 | #' 43 | #' @note 44 | #' `write_run_data()` is deprecated and is provided as an alias 45 | #' for backward compatibility. 46 | #' 47 | #' @keywords internal 48 | #' 49 | #' @export 50 | write_run_metadata <- function(type, data, run_dir = NULL) { 51 | 52 | # we need to create a write_fn so that the write can be deferred 53 | # until after a run_dir is actually established. Create the function 54 | # automatically for known types, for unknown types the `data` 55 | # argument is the write_fn 56 | 57 | # helper function to write simple named lists of values 58 | named_list_write_fn <- function(type) { 59 | function(data_dir) { 60 | jsonlite::write_json( 61 | data, 62 | path = file.path(data_dir, paste0(type, ".json")), 63 | auto_unbox = TRUE, # length-1 vectors as scalar 64 | pretty = TRUE, # formatted output 65 | force = TRUE, # flags as unclassed named list, 66 | digits = NA # save data with full precision 67 | ) 68 | } 69 | } 70 | 71 | # simple named list 72 | if (type %in% c("flags", "evaluation", "error")) { 73 | 74 | write_fn <- named_list_write_fn(type) 75 | 76 | # properties (written individually to support multiple writes) 77 | } else if (identical(type, "properties")) { 78 | 79 | write_fn <- function(data_dir) { 80 | properties_dir <- file.path(data_dir, "properties") 81 | if (!utils::file_test("-d", properties_dir)) 82 | dir.create(properties_dir, recursive = TRUE) 83 | for (name in names(data)) { 84 | property_file <- file.path(properties_dir, name) 85 | writeLines(as.character(data[[name]]), property_file) 86 | } 87 | } 88 | 89 | # metrics data frame 90 | } else if (identical(type, "metrics")) { 91 | 92 | write_fn <- function(data_dir) { 93 | write_metrics_json(data, file.path(data_dir, "metrics.json")) 94 | } 95 | 96 | # source code 97 | } else if (identical(type, "source")) { 98 | 99 | write_fn <- function(data_dir) { 100 | write_source_archive(data, data_dir, "source.tar.gz") 101 | } 102 | 103 | # custom type 104 | } else { 105 | if (!is.function(data)) 106 | stop("The data parameter must be a function for custom data types") 107 | write_fn <- data 108 | } 109 | 110 | # check for a run_dir. if we have one write the run data, otherwise 111 | # defer the write until we (maybe) acquire a run_dir later 112 | if (!is.null(run_dir)) 113 | write_fn(meta_dir(run_dir)) 114 | else if (is_run_active()) 115 | write_fn(meta_dir(run_dir())) 116 | else 117 | .globals$run_dir$pending_writes[[type]] <- write_fn 118 | 119 | # return nothing 120 | invisible(NULL) 121 | } 122 | 123 | write_run_property <- function(name, value) { 124 | properties <- list() 125 | properties[[name]] <- value 126 | write_run_metadata("properties", properties) 127 | } 128 | 129 | 130 | #' Write run data (deprecated) 131 | #' 132 | #' Deprecated alias for [write_run_metadata()]. 133 | #' 134 | #' @inheritParams write_run_metadata 135 | #' 136 | #' @keywords internal 137 | #' @export 138 | write_run_data <- function(type, data) { 139 | write_run_metadata(paste0("custom", type), data) 140 | } 141 | 142 | 143 | write_source_archive <- function(sources_dir, data_dir, archive) { 144 | 145 | # normalize paths since we'll be changing the working dir 146 | sources_dir <- normalizePath(sources_dir) 147 | data_dir <- normalizePath(data_dir) 148 | 149 | # change to sources_dir 150 | wd <- getwd() 151 | on.exit(setwd(wd), add = TRUE) 152 | setwd(sources_dir) 153 | 154 | # enumerate source files. note that we used to do this recursively but ran 155 | # into performance issues when script files were in a directory with large 156 | # subdirectories. Here's the commit where we backed this out: 157 | # https://github.com/rstudio/tfruns/commit/2dbcac627c82a2ecccecc2ba5ecada61d91255c7 158 | # (note that if we bring this back we need to continue ignoring the packrat dir) 159 | # the right solution might be to override `source` for the duration of the 160 | # run and just track which R scripts are sourced. 161 | files <- list.files(path = ".", 162 | pattern = utils::glob2rx("*.r"), 163 | ignore.case = TRUE) 164 | 165 | # create temp dir for sources 166 | sources_tmp_dir <- file.path(tempfile("tfruns-sources"), "source") 167 | on.exit(unlink(sources_tmp_dir), add = TRUE) 168 | dir.create(sources_tmp_dir, recursive = TRUE) 169 | 170 | # copy the sources to the temp dir 171 | for (file in files) { 172 | dir <- dirname(file) 173 | target_dir <- file.path(sources_tmp_dir, dir) 174 | if (!utils::file_test("-d", target_dir)) 175 | dir.create(target_dir, recursive = TRUE) 176 | file.copy(from = file, to = target_dir) 177 | } 178 | 179 | # create the tarball 180 | # tar and prevent "storing paths of more than 100 bytes is not 181 | # portable" warning issued by R 182 | setwd(file.path(sources_tmp_dir, "..")) 183 | suppressWarnings( 184 | utils::tar( 185 | file.path(data_dir, archive), 186 | files = "source", 187 | compression = "gzip", 188 | tar = "internal" 189 | ) 190 | ) 191 | } 192 | 193 | 194 | # get the meta dir for a run dir 195 | meta_dir <- function(run_dir, create = TRUE) { 196 | meta_dir <- file.path(run_dir, "tfruns.d") 197 | if (create && !utils::file_test("-d", meta_dir)) 198 | dir.create(meta_dir, recursive = TRUE) 199 | meta_dir 200 | } 201 | -------------------------------------------------------------------------------- /R/training_run.R: -------------------------------------------------------------------------------- 1 | #' Run a training script 2 | #' 3 | #' @inheritParams flags 4 | #' @inheritParams ls_runs 5 | #' @param file Path to training script (defaults to "train.R") 6 | #' @param context Run context (defaults to "local") 7 | #' @param flags Named list with flag values (see [flags()]) or path 8 | #' to YAML file containing flag values. 9 | #' @param properties Named character vector with run properties. Properties are 10 | #' additional metadata about the run which will be subsequently available via 11 | #' [ls_runs()]. 12 | #' @param run_dir Directory to store run data within 13 | #' @param artifacts_dir Directory to capture created and modified files within. 14 | #' Pass `NULL` to not capture any artifcats. 15 | #' @param echo Print expressions within training script 16 | #' @param view View the results of the run after training. The default "auto" 17 | #' will view the run when executing a top-level (printed) statement in an 18 | #' interactive session. Pass `TRUE` or `FALSE` to control whether the view is 19 | #' shown explictly. You can also pass "save" to save a copy of the 20 | #' run report at `tfruns.d/view.html` 21 | #' @param envir The environment in which the script should be evaluated 22 | #' @param encoding The encoding of the training script; see [file()]. 23 | #' 24 | #' @return Single row data frame with run flags, metrics, etc. 25 | #' 26 | #' @details The training run will by default use a unique new run directory 27 | #' within the "runs" sub-directory of the current working directory (or to the 28 | #' value of the `tfruns.runs_dir` R option if specified). 29 | #' 30 | #' The directory name will be a timestamp (in GMT time). If a duplicate name is 31 | #' generated then the function will wait long enough to return a unique one. 32 | #' 33 | #' If you want to use an alternate directory to store run data you can either 34 | #' set the global `tfruns.runs_dir` R option, or you can pass a `run_dir` 35 | #' explicitly to `training_run()`, optionally using the [unique_run_dir()] 36 | #' function to generate a timestamp-based directory name. 37 | #' 38 | #' @export 39 | training_run <- function(file = "train.R", 40 | context = "local", 41 | config = Sys.getenv("R_CONFIG_ACTIVE", unset = "default"), 42 | flags = NULL, 43 | properties = NULL, 44 | run_dir = NULL, 45 | artifacts_dir = getwd(), 46 | echo = TRUE, 47 | view = "auto", 48 | envir = parent.frame(), 49 | encoding = getOption("encoding")) { 50 | 51 | # if file is not specified then see if there is a single R source file 52 | # within the current working directory 53 | if (missing(file)) { 54 | all_r_scripts <- list.files(pattern = utils::glob2rx("*.r"), ignore.case = TRUE) 55 | if (length(all_r_scripts) == 1) 56 | file <- all_r_scripts 57 | } 58 | 59 | # verify that the file exists 60 | if (!file.exists(file)) 61 | stop("The R script '", file, "' does not exist.") 62 | 63 | # setup run context 64 | run_dir <- initialize_run( 65 | file = file, 66 | type = "training", 67 | context = context, 68 | config = config, 69 | flags = flags, 70 | properties = properties, 71 | run_dir = run_dir 72 | ) 73 | 74 | # execute the training run 75 | do_training_run(file, run_dir, artifacts_dir, echo = echo, envir = envir, encoding = encoding) 76 | 77 | # check for forced view 78 | force_view <- isTRUE(view) 79 | 80 | # result "auto" if necessary 81 | if (identical(view, "auto")) 82 | view <- interactive() 83 | 84 | # print completed message 85 | message('\nRun completed: ', run_dir, '\n') 86 | 87 | # prepare to return the run 88 | run_return <- return_runs(run_record(run_dir)) 89 | 90 | # force_view means we do the view (i.e. we don't rely on printing) 91 | if (force_view) { 92 | 93 | view_run(run_dir) 94 | invisible(run_return) 95 | 96 | # regular view means give it a class that will result in a view 97 | # when executed as a top-level statement 98 | } else if (isTRUE(view)) { 99 | 100 | class(run_return) <- c("tfruns_viewed_run", class(run_return)) 101 | run_return 102 | 103 | # save a copy of the run view 104 | } else if (identical(view, "save")) { 105 | 106 | save_run_view(run_dir, file.path(run_dir, "tfruns.d", "view.html")) 107 | invisible(run_return) 108 | 109 | # otherwise just return invisibly 110 | } else { 111 | 112 | invisible(run_return) 113 | 114 | } 115 | } 116 | 117 | 118 | #' Tune hyperparameters using training flags 119 | #' 120 | #' Run all combinations of the specifed training flags. The number of 121 | #' combinations can be reduced by specifying the `sample` parameter, which 122 | #' will result in a random sample of the flag combinations being run. 123 | #' 124 | #' @inheritParams training_run 125 | #' @inheritParams ls_runs 126 | #' 127 | #' @param flags Either a named list with flag values (multiple values can be 128 | #' provided for each flag) or a data frame that contains pre-generated 129 | #' combinations of flags (e.g. via [base::expand.grid()]). The latter can 130 | #' be useful for subsetting combinations. See 'Examples'. 131 | #' @param sample Sampling rate for flag combinations (defaults to 132 | #' running all combinations). 133 | #' @param confirm Confirm before executing tuning run. 134 | #' 135 | #' @return Data frame with summary of all training runs performed 136 | #' during tuning. 137 | #' 138 | #' @examples 139 | #' \dontrun{ 140 | #' library(tfruns) 141 | #' 142 | #' # using a list as input to the flags argument 143 | #' runs <- tuning_run( 144 | #' system.file("examples/mnist_mlp/mnist_mlp.R", package = "tfruns"), 145 | #' flags = list( 146 | #' dropout1 = c(0.2, 0.3, 0.4), 147 | #' dropout2 = c(0.2, 0.3, 0.4) 148 | #' ) 149 | #' ) 150 | #' runs[order(runs$eval_acc, decreasing = TRUE), ] 151 | #' 152 | #' # using a data frame as input to the flags argument 153 | #' # resulting in the same combinations above, but remove those 154 | #' # where the combined dropout rate exceeds 1 155 | #' grid <- expand.grid( 156 | #' dropout1 = c(0.2, 0.3, 0.4), 157 | #' dropout2 = c(0.2, 0.3, 0.4) 158 | #' ) 159 | #' grid$combined_droput <- grid$dropout1 + grid$dropout2 160 | #' grid <- grid[grid$combined_droput <= 1, ] 161 | #' runs <- tuning_run( 162 | #' system.file("examples/mnist_mlp/mnist_mlp.R", package = "tfruns"), 163 | #' flags = grid[, c("dropout1", "dropout2")] 164 | #' ) 165 | #' } 166 | #' @export 167 | tuning_run <- function(file = "train.R", 168 | context = "local", 169 | config = Sys.getenv("R_CONFIG_ACTIVE", unset = "default"), 170 | flags = NULL, 171 | sample = NULL, 172 | properties = NULL, 173 | runs_dir = getOption("tfruns.runs_dir", "runs"), 174 | artifacts_dir = getwd(), 175 | echo = TRUE, 176 | confirm = interactive(), 177 | envir = parent.frame(), 178 | encoding = getOption("encoding")) { 179 | 180 | if (!is.list(flags) || is.null(names(flags))) { 181 | stop("flags must be specified as a named list or a data frame") 182 | } 183 | 184 | 185 | # set runs dir if specified 186 | old_runs_dir <- getOption("tfruns.runs_dir") 187 | options(tfruns.runs_dir = runs_dir) 188 | if (!is.null(old_runs_dir)) 189 | on.exit(options(tfruns.runs_dir = old_runs_dir), add = TRUE) 190 | 191 | # calculate the flag grid 192 | if (!is.data.frame(flags)) { 193 | flag_grid <- do.call(function(...) expand.grid(..., stringsAsFactors = FALSE), flags) 194 | message(prettyNum(nrow(flag_grid), big.mark = ","), " total combinations of flags ") 195 | } else { 196 | flag_grid <- flags 197 | } 198 | 199 | # sample if requested 200 | if (!is.null(sample)) { 201 | if (sample > 1) 202 | stop("sample must be a floating point value less or equal to 1") 203 | if (sample <= 0) 204 | stop("sample must be a floating point value greater than 0") 205 | indices <- sample.int(nrow(flag_grid), size = ceiling(sample * nrow(flag_grid))) 206 | flag_grid <- flag_grid[indices, , drop = FALSE] 207 | message("(sampled to ", prettyNum(nrow(flag_grid), big.mark = ","), " combinations)\n") 208 | } else { 209 | message("(use sample parameter to run a random subset)") 210 | } 211 | 212 | if (confirm) { 213 | prompt <- readline("Proceed with tuning run? [Y/n]: ") 214 | if (nzchar(prompt) && tolower(prompt) != 'y') 215 | return(invisible(NULL)) 216 | } 217 | 218 | # execute tuning 219 | for (i in 1:nrow(flag_grid)) { 220 | 221 | # flags 222 | flags = as.list(flag_grid[i,, drop = FALSE]) 223 | message(sprintf( 224 | "Training run %d/%d (flags = %s) ", 225 | i, 226 | nrow(flag_grid), 227 | deparse(flags, control = c("keepInteger", "keepNA")) 228 | )) 229 | 230 | # execute run 231 | training_run( 232 | file = file, 233 | config = config, 234 | flags = flags, 235 | properties = properties, 236 | run_dir = NULL, 237 | artifacts_dir = artifacts_dir, 238 | echo = echo, 239 | view = FALSE, 240 | envir = new.env(parent = envir), 241 | encoding = encoding 242 | ) 243 | } 244 | 245 | # return data frame with all runs 246 | invisible(ls_runs(latest_n = nrow(flag_grid), runs_dir = runs_dir)) 247 | } 248 | 249 | 250 | #' @export 251 | print.tfruns_viewed_run <- function(x, ...) { 252 | view_run(x) 253 | } 254 | 255 | 256 | do_training_run <- function(file, run_dir, artifacts_dir, echo, envir, encoding) { 257 | 258 | with_changed_file_copy(artifacts_dir, run_dir, { 259 | 260 | # write script 261 | write_run_property("script", basename(file)) 262 | 263 | # write begin and end times 264 | write_run_property("start", as.double(Sys.time())) 265 | on.exit(write_run_property("end", as.double(Sys.time())), add = TRUE) 266 | 267 | # clear run on exit 268 | on.exit(clear_run(), add = TRUE) 269 | 270 | # clear TF session on exist 271 | on.exit(reset_tf_graph(), add = TRUE) 272 | 273 | # set width for run 274 | old_width <- getOption("width") 275 | options(width = min(100, old_width)) 276 | on.exit(options(width = old_width), add = TRUE) 277 | 278 | # sink output 279 | properties_dir <- file.path(meta_dir(run_dir), "properties") 280 | output_file <- file(file.path(properties_dir, "output"), open = "wt", encoding = "UTF-8") 281 | sink(file = output_file, type = "output", split = TRUE) 282 | on.exit({ sink(type = "output"); close(output_file); }, add = TRUE) 283 | 284 | # sink plots 285 | plots_dir <- file.path(run_dir, "plots") 286 | if (!utils::file_test("-d", plots_dir)) 287 | dir.create(plots_dir, recursive = TRUE) 288 | png_args <- list( 289 | filename = file.path(plots_dir, "Rplot%03d.png"), 290 | width=1200, height=715, res = 192 # ~ golden ratio @ highdpi 291 | ) 292 | if (is_windows() && capabilities("cairo")) # required to prevent empty plot 293 | png_args$type <- "cairo" # emitted when type = "windows" 294 | do.call(grDevices::png, png_args) 295 | dev_number <- grDevices::dev.cur() 296 | on.exit(grDevices::dev.off(dev_number), add = TRUE) 297 | 298 | # notify user of run dir 299 | message("Using run directory ", run_dir) 300 | 301 | # perform the run 302 | write_run_property("completed", FALSE) 303 | withCallingHandlers({ 304 | source(file = file, local = envir, echo = echo, encoding = encoding) 305 | write_run_property("completed", TRUE) 306 | }, 307 | error = function(e) { 308 | 309 | # write error 310 | write_run_metadata("error", list( 311 | message = e$message, 312 | traceback = capture_stacktrace(sys.calls()) 313 | )) 314 | 315 | # forward error 316 | stop(e) 317 | }) 318 | }) 319 | } 320 | 321 | 322 | initialize_run <- function(file, 323 | type = "training", 324 | context = "local", 325 | config = Sys.getenv("R_CONFIG_ACTIVE", unset = "default"), 326 | flags = NULL, 327 | properties = NULL, 328 | run_dir = NULL) { 329 | 330 | # clear any existing run 331 | clear_run() 332 | 333 | # generate the run_dir 334 | if (is.null(run_dir)) 335 | run_dir <- unique_run_dir() 336 | 337 | # create the directory if necessary 338 | if (!utils::file_test("-d", run_dir)) 339 | if (!dir.create(run_dir, recursive = TRUE)) 340 | stop("Unable to create run directory at ", run_dir) 341 | 342 | # if flags is a YAML file then read the flags from the file 343 | if (is.character(flags) && length(flags) == 1 && 344 | is.null(names(flags)) && file.exists(flags)) { 345 | flags_file <- flags 346 | flags <- NULL 347 | } else { 348 | flags_file <- NULL 349 | } 350 | 351 | # this is new definition for the run_dir, save it 352 | .globals$run_dir$path <- run_dir 353 | 354 | # save config and flags (they'll get processed later in flags()) 355 | .globals$run_dir$config <- config 356 | .globals$run_dir$flags <- flags 357 | .globals$run_dir$flags_file <- flags_file 358 | 359 | # write type and context 360 | write_run_metadata("properties", list( 361 | type = type, 362 | context = context 363 | )) 364 | 365 | # write properties 366 | write_run_metadata("properties", properties) 367 | 368 | # write source files 369 | write_run_metadata("source", dirname(file)) 370 | 371 | # execute any pending writes 372 | for (name in ls(.globals$run_dir$pending_writes)) 373 | .globals$run_dir$pending_writes[[name]](meta_dir(run_dir)) 374 | 375 | # return invisibly 376 | invisible(run_dir) 377 | } 378 | 379 | clear_run <- function() { 380 | .globals$run_dir$path <- NULL 381 | .globals$run_dir$config <- NULL 382 | .globals$run_dir$flags <- NULL 383 | .globals$run_dir$flags_file <- NULL 384 | .globals$run_dir$pending_writes <- new.env(parent = emptyenv()) 385 | } 386 | 387 | reset_tf_graph <- function() { 388 | tryCatch({ 389 | if (reticulate::py_module_available("tensorflow")) { 390 | tf <- reticulate::import("tensorflow") 391 | if (tf_version(tf) >= "1.13" && !tf$executing_eagerly()) 392 | if(tf_version(tf) >= "2.0") { 393 | tf$compat$v1$reset_default_graph() 394 | } else { 395 | tf$reset_default_graph() 396 | } 397 | if (reticulate::py_has_attr(tf, "keras")) 398 | tf$keras$backend$clear_session() 399 | else if (reticulate::py_has_attr(tf$contrib, "keras")) 400 | tf$contrib$keras$backend$clear_session() 401 | } 402 | if (reticulate::py_module_available("keras")) { 403 | keras <- reticulate::import("keras") 404 | if (reticulate::py_has_attr(keras$backend, "clear_session")) 405 | keras$backend$clear_session() 406 | } 407 | }, error = function(e) { 408 | warning("Error occurred resetting tf graph: ", e$message) 409 | }) 410 | } 411 | 412 | capture_stacktrace <- function(stack) { 413 | stack <- stack[-(2:7)] 414 | stack <- utils::head(stack, -2) 415 | stack <- vapply(stack, function(frame) { 416 | frame <- deparse(frame) 417 | frame <- paste(frame, collapse = "\n") 418 | frame 419 | }, FUN.VALUE = "frame") 420 | rev(stack) 421 | } 422 | 423 | with_changed_file_copy <- function(artifacts_dir, run_dir, expr) { 424 | 425 | if (!is.null(artifacts_dir)) { 426 | # snapshot the files in the training script directory before training 427 | files_before <- utils::fileSnapshot(path = artifacts_dir, recursive = TRUE) 428 | 429 | # copy changed files on exit 430 | on.exit({ 431 | # snapshot the files in the run_dir after training then compute changed files 432 | files_after <- utils::fileSnapshot(path = artifacts_dir, recursive = TRUE) 433 | changed_files <- utils::changedFiles(files_before, files_after) 434 | 435 | # filter out files within the run_dir and packrat/gs files (cloudml) 436 | changed_files <- c(changed_files$changed, changed_files$added) 437 | changed_files <- changed_files[!grepl(basename(run_dir), changed_files)] 438 | changed_files <- changed_files[!grepl("^packrat[/\\]", changed_files)] 439 | changed_files <- changed_files[!grepl("^gs[/\\]", changed_files)] 440 | 441 | # copy the changed files to the run_dir 442 | for (file in changed_files) { 443 | dir <- dirname(file) 444 | target_dir <- file.path(run_dir, dir) 445 | if (!utils::file_test("-d", target_dir)) 446 | dir.create(target_dir, recursive = TRUE) 447 | file.copy(from = file.path(artifacts_dir, file), to = target_dir) 448 | } 449 | }, add = TRUE) 450 | } 451 | 452 | # execute the expression 453 | force(expr) 454 | } 455 | 456 | 457 | #' Save a run view as HTML 458 | #' 459 | #' The saved view includes summary information (flags, metrics, model 460 | #' attributes, etc.), plot and console output, and the code used for the run. 461 | #' 462 | #' @inheritParams run_info 463 | #' @param filename Path to save the HTML to. If no `filename` is specified 464 | #' then a temporary file is used (the path to the file is returned invisibly). 465 | #' 466 | #' @seealso [ls_runs()], [run_info()], [view_run()] 467 | #' 468 | #' @import base64enc 469 | #' 470 | #' @export 471 | save_run_view <- function(run_dir = latest_run(), filename = "auto") { 472 | 473 | # verify run_dir 474 | if (is.null(run_dir)) 475 | stop("No runs available in the current directory") 476 | 477 | # get run view data 478 | run <- run_info(run_dir) 479 | data <- run_view_data(run) 480 | 481 | # generate filename if needed 482 | if (identical(filename, "auto")) 483 | filename <- viewer_temp_file(paste0("run-", basename(run$run_dir))) 484 | 485 | # save the report 486 | save_page("view_run", data = data, filename) 487 | 488 | # return the path saved to 489 | invisible(filename) 490 | } 491 | 492 | 493 | 494 | #' View a training run 495 | #' 496 | #' View metrics and other attributes of a training run. 497 | #' 498 | #' @inheritParams run_info 499 | #' @param viewer Viewer to display training run information within 500 | #' (default to an internal page viewer if available, otherwise 501 | #' to the R session default web browser). 502 | #' 503 | #' @seealso [ls_runs()], [run_info()] 504 | #' 505 | #' @import base64enc 506 | #' 507 | #' @export 508 | view_run <- function(run_dir = latest_run(), viewer = getOption("tfruns.viewer")) { 509 | 510 | # verify run_dir 511 | if (is.null(run_dir)) 512 | stop("No runs available in the current directory") 513 | 514 | # generate run report and view it 515 | view_page(save_run_view(run_dir), viewer) 516 | } 517 | 518 | #' Save a run comparison as HTML 519 | #' 520 | #' @inheritParams save_run_view 521 | #' 522 | #' @param runs Character vector of 2 training run directories or 523 | #' data frame returned from [ls_runs()] with at least 2 elements. 524 | #' 525 | #' @export 526 | save_run_comparison <- function(runs = ls_runs(latest_n = 2), filename = "auto") { 527 | 528 | # cast to run_info 529 | runs <- run_info(runs) 530 | 531 | # verify exactly 2 runs provided 532 | if (length(runs) != 2) 533 | stop("You must pass at least 2 run directories to compare_runs") 534 | 535 | # generate filename if needed 536 | if (identical(filename, "auto")) { 537 | filename <- viewer_temp_file(paste("compare-runs", 538 | basename(runs[[1]]$run_dir), 539 | basename(runs[[2]]$run_dir), 540 | sep = "-")) 541 | } 542 | 543 | # runs to compare (order least to most recent) 544 | if (runs[[1]]$start > runs[[2]]$start) { 545 | run_a <- runs[[2]] 546 | run_b <- runs[[1]] 547 | } else { 548 | run_a <- runs[[1]] 549 | run_b <- runs[[2]] 550 | } 551 | 552 | # get view_data 553 | run_a <- run_view_data(run_a) 554 | run_b <- run_view_data(run_b) 555 | 556 | # generate diffs 557 | diffs <- run_diffs(run_a, run_b) 558 | 559 | # data for view 560 | data <- list(run_a = run_a, run_b = run_b, diffs = diffs) 561 | 562 | # save the report 563 | save_page("compare_runs", data = data, filename) 564 | 565 | # return the path saved to 566 | invisible(filename) 567 | } 568 | 569 | #' Compare training runs 570 | #' 571 | #' Render a visual comparison of two training runs. The runs are 572 | #' displayed with the most recent run on the right and the 573 | #' earlier run on the left. 574 | #' 575 | #' @inheritParams view_run 576 | #' @inheritParams save_run_comparison 577 | #' 578 | #' @export 579 | compare_runs <- function(runs = ls_runs(latest_n = 2), 580 | viewer = getOption("tfruns.viewer")) { 581 | 582 | # verify runs 583 | if (is.null(runs)) 584 | stop("No runs available in the current directory") 585 | 586 | # save and view 587 | view_page(save_run_comparison(runs), viewer) 588 | } 589 | 590 | run_view_data <- function(run) { 591 | 592 | # helper to extract prefaced properties 593 | with_preface <- function(preface, strip_preface = TRUE) { 594 | preface_pattern <- paste0("^", preface, "_") 595 | prefaced <- run[grepl(preface_pattern, names(run))] 596 | if (length(prefaced) > 0) { 597 | if (strip_preface) 598 | names(prefaced) <- sub(preface_pattern, "", names(prefaced)) 599 | prefaced 600 | } else { 601 | NULL 602 | } 603 | } 604 | 605 | # helpers for formatting numbers 606 | format_integer <- function(x) { 607 | if (!is.null(x)) 608 | prettyNum(x, mode = "integer", big.mark = ",") 609 | else 610 | NULL 611 | } 612 | format_numeric <- function(x, digits = 4) { 613 | if (!is.null(x)) 614 | formatC(x, format='f', digits= digits) 615 | else 616 | NULL 617 | } 618 | 619 | # default some potentially empty sections to null 620 | data <- list( 621 | cloudml = NULL, 622 | history = NULL, 623 | model = NULL, 624 | metrics = NULL, 625 | evaluation = NULL, 626 | optimization = NULL, 627 | training = NULL, 628 | flags = NULL, 629 | files = NULL, 630 | plots = NULL, 631 | output = NULL, 632 | error = NULL 633 | ) 634 | 635 | # run_dir 636 | data$run_dir <- basename(run$run_dir) 637 | 638 | # attributes 639 | script <- basename(run$script) 640 | data$attributes <- list( 641 | context = run$context, 642 | script = script, 643 | started = paste(as.POSIXct(run$start, origin="1970-01-01", tz = "GMT"), 644 | "GMT"), 645 | time = format(as.POSIXct(as.character(Sys.Date()), tz = "GMT") + 646 | run$end - run$start, 647 | "%H:%M:%S") 648 | ) 649 | 650 | # cloudml attributes 651 | if (identical(run$context, "cloudml")) { 652 | cloudml <- with_preface("cloudml") 653 | if (!is.null(cloudml)) { 654 | data$cloudml <- list() 655 | data$cloudml$job <- list( 656 | href = cloudml$console_url, 657 | text = cloudml$job 658 | ) 659 | data$cloudml$logs <- list( 660 | href = cloudml$log_url, 661 | text = "View logs" 662 | ) 663 | data$cloudml$master_type <- cloudml$master_type 664 | data$cloudml$status <- cloudml$state 665 | data$cloudml$created <- paste( 666 | as.POSIXct(as.numeric(cloudml$created), 667 | origin="1970-01-01", tz = "GMT"), 668 | "GMT" 669 | ) 670 | data$cloudml$time <- format(as.POSIXct(as.character(Sys.Date()), tz = "GMT") + 671 | as.numeric(cloudml$end) - as.numeric(cloudml$created), 672 | "%H:%M:%S") 673 | data$cloudml$ml_units <- cloudml$ml_units 674 | } 675 | } 676 | 677 | # metrics 678 | metrics <- with_preface("metric") 679 | if (!is.null(metrics)) 680 | data$metrics <- lapply(metrics, format_numeric) 681 | 682 | # evaluation 683 | evaluation <- with_preface("eval", strip_preface = FALSE) 684 | if (!is.null(evaluation)) 685 | data$evaluation <- evaluation 686 | 687 | # flags 688 | flags <- with_preface("flag") 689 | if (!is.null(flags)) 690 | data$flags <- flags 691 | 692 | # optimization 693 | optimization <- list() 694 | optimization$loss <- run$loss_function 695 | optimization$optimizer <- run$optimizer 696 | optimization$lr <- run$learning_rate 697 | if (length(optimization) > 0) 698 | data$optimization <- optimization 699 | 700 | # training 701 | 702 | # determine the step units 703 | steps_unit <- get_steps_unit(run) 704 | steps_completed_unit <- get_steps_completed_unit(steps_unit) 705 | 706 | # format the steps 707 | if (!is.null(run[[steps_unit]]) && !is.null(run[[steps_completed_unit]])) { 708 | if (run[[steps_unit]] > run[[steps_completed_unit]]) 709 | steps <- paste(format_integer(run[[steps_completed_unit]]), 710 | format_integer(run[[steps_unit]]), 711 | sep = "/") 712 | else 713 | steps <- format_integer(run[[steps_unit]]) 714 | } else { 715 | steps <- NULL 716 | } 717 | training <- list() 718 | training$samples <- format_integer(run$samples) 719 | training$validation_samples <- format_integer(run$validation_samples) 720 | training[[steps_unit]] <- steps 721 | training$batch_size <- format_integer(run$batch_size) 722 | if (length(training) > 0) 723 | data$training <- training 724 | 725 | # error 726 | if (!is.null(run$error_message)) { 727 | data$error <- list( 728 | message = run$error_message, 729 | traceback = run$error_traceback 730 | ) 731 | } 732 | 733 | # metrics history 734 | if (!is.null(run$metrics)) 735 | data$history <- run$metrics 736 | 737 | # model summary 738 | if (!is.null(run$model)) { 739 | data$model <- sub("^Model\n", "", run$model) 740 | data$model <- sub("^_+\n", "", data$model) 741 | } 742 | 743 | # initialize tabs 744 | data$tabs <- list( 745 | list(href = "#summary", title = "Summary"), 746 | list(href = "#output", title = "Output"), 747 | list(href = "#code", title = "Code") 748 | ) 749 | 750 | # determine if we have an output tab (remove it if we don't) 751 | data$output_tab <- !is.null(data$error) || 752 | !is.null(data$history) || 753 | !is.null(data$model) 754 | if (!data$output_tab) 755 | data$tabs[[2]] <- NULL 756 | 757 | # source code 758 | data$source_code <- run_source_code(script, run$run_dir) 759 | 760 | # files 761 | files <- list.files(run$run_dir, recursive = FALSE) 762 | files <- gsub("\\\\", "/", files) 763 | files <- files[!files %in% c("tfruns.d", "plots", "logs")] 764 | if (length(files) > 0) { 765 | data$files <- lapply(files, function(file) { 766 | list( 767 | name = file, 768 | directory = utils::file_test("-d", file.path(run$run_dir, file)) 769 | ) 770 | }) 771 | } 772 | 773 | # plots 774 | plots <- list.files(file.path(run$run_dir, "plots"), 775 | pattern = utils::glob2rx("*.png"), 776 | full.names = TRUE) 777 | if (length(plots) > 0) { 778 | data$plots <- lapply(plots, function(plot) { 779 | base64enc::dataURI(file = plot, 780 | mime = "image/png", 781 | encoding = "base64") 782 | }) 783 | } 784 | 785 | # output 786 | if (!is.null(run$output)) 787 | data$output <- run$output 788 | 789 | # return data 790 | data 791 | } 792 | 793 | 794 | run_diffs <- function(run_a, run_b) { 795 | 796 | diffs <- list() 797 | 798 | # FLAGS 799 | run_flags <- function(run) ifelse(is.null(run$flags), "", yaml::as.yaml(run$flags)) 800 | run_a_flags <- run_flags(run_a) 801 | run_b_flags <- run_flags(run_b) 802 | if (!identical(run_a_flags, run_b_flags)) { 803 | diffs[["FLAGS"]] <- list( 804 | run_a = run_a_flags, 805 | run_b = run_b_flags 806 | ) 807 | } 808 | 809 | # source code: changes and deletions 810 | for (source_file in names(run_a$source_code)) { 811 | run_a_source_file <- run_a$source_code[[source_file]] 812 | run_b_source_file <- run_b$source_code[[source_file]] 813 | if (is.null(run_b_source_file)) 814 | run_b_source_file <- "" 815 | if (!identical(run_a_source_file, run_b_source_file)) { 816 | diffs[[source_file]] <- list( 817 | run_a = run_a_source_file, 818 | run_b = run_b_source_file 819 | ) 820 | } 821 | } 822 | # source code: additions 823 | new_source_files <- setdiff(names(run_b$source_code), names(run_a$source_code)) 824 | for (source_file in new_source_files) { 825 | diffs[[source_file]] <- list( 826 | run_a = "", 827 | run_b = run_b$source_code[[source_file]] 828 | ) 829 | } 830 | 831 | # return diffs 832 | diffs 833 | } 834 | 835 | 836 | 837 | -------------------------------------------------------------------------------- /R/utils.R: -------------------------------------------------------------------------------- 1 | is_windows <- function() { 2 | identical(.Platform$OS.type, "windows") 3 | } 4 | 5 | # we cant use tensorflow::tf_version() to not create a 6 | # dependency 7 | tf_version <- function(tf) { 8 | 9 | if (reticulate::py_has_attr(tf, "version")) 10 | version_raw <- tf$version$VERSION 11 | else version_raw <- tf$VERSION 12 | 13 | tfv <- strsplit(version_raw, ".", fixed = TRUE)[[1]] 14 | version <- package_version(paste(tfv[[1]], tfv[[2]], 15 | sep = ".")) 16 | 17 | version 18 | } 19 | -------------------------------------------------------------------------------- /R/views.R: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | #' @import whisker 5 | render_view <- function(view, output_file, variables = list()) { 6 | 7 | # read a component 8 | read_component <- function(name) { 9 | 10 | if (!exists(name, envir = .globals$view_components)) { 11 | lines <- readLines(system.file("views", "components", paste0(name, ".html"), 12 | package = "tfruns"), 13 | encoding = "UTF-8") 14 | component <- paste(lines, collapse = "\n") 15 | assign(name, component, envir = .globals$view_components) 16 | } 17 | 18 | get(name, envir = .globals$view_components) 19 | } 20 | 21 | # add components to variables 22 | variables <- append(variables, list( 23 | jquery = read_component("jquery"), 24 | roboto = read_component("roboto"), 25 | material_icons = read_component("material_icons"), 26 | materialize = read_component("materialize"), 27 | vue_js = read_component("vue_min_js"), 28 | c3 = read_component("c3"), 29 | metrics_charts = read_component("metrics_charts"), 30 | highlight_js = read_component("highlight_js"), 31 | diff2html = read_component("diff2html"), 32 | dashboard = read_component("dashboard") 33 | )) 34 | 35 | if (view == "metrics") { 36 | # then we are running on rstudio and we want to match backgrounds and etc 37 | theme_info <- rstudioapi::getThemeInfo() 38 | variables[["background_color"]] <- theme_info$background 39 | variables[["foreground_color"]] <- theme_info$foreground 40 | variables[["font_size"]] <- paste0( 41 | rstudioapi::readRStudioPreference("help_font_size_points", 12), 42 | "px" 43 | ) 44 | } 45 | 46 | # read the template 47 | template <- readLines(system.file("views", paste0(view, ".html"), package = "tfruns"), 48 | encoding = "UTF-8") 49 | 50 | # render the template 51 | html <- whisker.render(template = template, data = variables) 52 | 53 | # write it 54 | writeLines(html, output_file, useBytes = TRUE) 55 | } 56 | 57 | 58 | save_page <- function(page, data, target_file) { 59 | 60 | # convert data to json 61 | data_json <- jsonlite::toJSON(data, 62 | dataframe = "columns", 63 | na = "null", 64 | null = "null", 65 | auto_unbox = TRUE) 66 | 67 | # render it 68 | render_view(page, target_file, list(data = data_json)) 69 | 70 | } 71 | 72 | viewer_temp_file <- function(stem) { 73 | 74 | viewer_dir <- tempfile("tfruns-") 75 | dir.create(viewer_dir) 76 | file.path(viewer_dir, paste0(stem, ".html")) 77 | 78 | } 79 | 80 | view_page <- function(viewer_html, viewer) { 81 | 82 | if (is.null(viewer)) 83 | viewer <- getOption("page_viewer", default = utils::browseURL) 84 | if (identical(viewer, getOption("page_viewer"))) { 85 | args <- list(url = viewer_html) 86 | if (!is.null(formals(viewer)$self_contained)) 87 | args$self_contained <- TRUE 88 | do.call(viewer, args) 89 | } else { 90 | browser_viewer(dirname(viewer_html), viewer)(viewer_html) 91 | } 92 | 93 | invisible(viewer_html) 94 | 95 | } 96 | 97 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | tfruns: Track, Visualize, and Manage Training Runs 2 | ================ 3 | 4 | 5 | [![R-CMD-check](https://github.com/rstudio/tfruns/workflows/R-CMD-check/badge.svg)](https://github.com/rstudio/tfruns/actions) 6 | 7 | 8 | Overview 9 | -------- 10 | 11 | The **tfruns** package provides a suite of tools for managing TensorFlow training runs and experiments from R: 12 | 13 | - Track the hyperparameters, metrics, output, and source code of every training run. 14 | 15 | - Compare hyperparameters and metrics across runs to find the best performing model. 16 | 17 | - Automatically generate reports to visualize individual training runs or comparisons between runs. 18 | 19 | - No changes to source code required (run data is automatically captured for all Keras and TF Estimator models). 20 | 21 | You can find documentation for the tfruns package at 22 | -------------------------------------------------------------------------------- /cran-comments.md: -------------------------------------------------------------------------------- 1 | * New release 2 | 3 | -------------------------------------------------------------------------------- /dev/boston_housing/train.R: -------------------------------------------------------------------------------- 1 | library(keras) 2 | 3 | FLAGS = flags( 4 | flag_integer('epochs1', 10) 5 | ) 6 | 7 | data = dataset_boston_housing(path = "boston_housing.npz", test_split = 0.2, 8 | seed = 113L) 9 | # for simplicity 10 | trainx = data$train$x 11 | trainy = data$train$y 12 | testx = data$test$x 13 | testy = data$test$y 14 | model1 = keras_model_sequential(input_shape = 13) 15 | layer_dense(object = model1, units = 5, activation = 'relu') 16 | layer_dense(model1, units = 3, activation = 'relu') 17 | layer_dense(model1, units = 1) 18 | compile(model1, optimizer = 'rmsprop', loss = 'mse', metrics='mae') 19 | trainx=normalize(data$train$x) # normalizes values to (0,1) trainy=keras_array(data$train$y) # produces a Python-formatted array testx=normalize(data$test$x) 20 | # the training step 21 | history = fit(model1,trainx,trainy, epochs=FLAGS$epochs1) 22 | # with test data 23 | predi = predict(model1, testx) 24 | MAE = mean(abs(predi[,1] - predi[,1]*testy)) # un-normalizing again MAE 25 | score <- model1 %>% evaluate( testx, testy, 26 | verbose = 0 ) 27 | cat('Test loss:', score$loss, '\n') 28 | cat('Test accuracy:', score$mean_absolute_error, '\n') 29 | 30 | 31 | -------------------------------------------------------------------------------- /dev/flags.R: -------------------------------------------------------------------------------- 1 | 2 | library(tfruns) 3 | 4 | FLAGS <- flags( 5 | flag_integer("integer", 1:5), 6 | flag_boolean("boolean", TRUE) 7 | ) 8 | -------------------------------------------------------------------------------- /dev/mnist_mlp/.gitignore: -------------------------------------------------------------------------------- 1 | *.h5 2 | dropout_tuning 3 | -------------------------------------------------------------------------------- /dev/mnist_mlp/mnist_mlp.R: -------------------------------------------------------------------------------- 1 | #' Trains a simple deep NN on the MNIST dataset. 2 | #' 3 | #' Gets to 98.40% test accuracy after 20 epochs 4 | #' (there is *a lot* of margin for parameter tuning). 5 | #' 2 seconds per epoch on a K520 GPU. 6 | #' 7 | 8 | library(keras) 9 | 10 | FLAGS <- flags( 11 | flag_integer("batch_size", default = 128), 12 | flag_numeric("dropout1", default = 0.4), 13 | flag_numeric("dropout2", default = 0.3) 14 | ) 15 | 16 | num_classes <- 10 17 | epochs <- 30 18 | 19 | # the data, shuffled and split between train and test sets 20 | mnist <- dataset_mnist() 21 | x_train <- mnist$train$x 22 | y_train <- mnist$train$y 23 | x_test <- mnist$test$x 24 | y_test <- mnist$test$y 25 | 26 | dim(x_train) <- c(nrow(x_train), 784) 27 | dim(x_test) <- c(nrow(x_test), 784) 28 | 29 | x_train <- x_train / 255 30 | x_test <- x_test / 255 31 | 32 | cat(nrow(x_train), 'train samples\n') 33 | cat(nrow(x_test), 'test samples\n') 34 | 35 | # convert class vectors to binary class matrices 36 | y_train <- to_categorical(y_train, num_classes) 37 | y_test <- to_categorical(y_test, num_classes) 38 | 39 | model <- keras_model_sequential() 40 | model %>% 41 | layer_dense(units = 128, activation = 'relu', input_shape = c(784)) %>% 42 | layer_dropout(rate = FLAGS$dropout1) %>% 43 | layer_dense(units = 128, activation = 'relu') %>% 44 | layer_dropout(rate = FLAGS$dropout2) %>% 45 | layer_dense(units = num_classes, activation = 'softmax') 46 | 47 | model %>% compile( 48 | loss = 'categorical_crossentropy', 49 | optimizer = optimizer_rmsprop(lr = 0.001), 50 | metrics = c('accuracy') 51 | ) 52 | 53 | history <- model %>% fit( 54 | x_train, y_train, 55 | batch_size = FLAGS$batch_size, 56 | epochs = epochs, 57 | verbose = 1, 58 | validation_split = 0.2 59 | ) 60 | 61 | plot(history) 62 | 63 | score <- model %>% evaluate( 64 | x_test, y_test, 65 | verbose = 0 66 | ) 67 | 68 | cat('Test loss:', score[[1]], '\n') 69 | cat('Test accuracy:', score[[2]], '\n') 70 | 71 | # save the model 72 | save_model_hdf5(model, "model.h5") 73 | save_model_weights_hdf5(model, 'model_weights.h5') 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /images/compare_runs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/images/compare_runs.png -------------------------------------------------------------------------------- /images/ls_runs_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/images/ls_runs_compare.png -------------------------------------------------------------------------------- /images/ls_runs_rstudio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/images/ls_runs_rstudio.png -------------------------------------------------------------------------------- /images/rstudio_addin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/images/rstudio_addin.png -------------------------------------------------------------------------------- /images/rstudio_publish.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/images/rstudio_publish.png -------------------------------------------------------------------------------- /images/rstudio_terminal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/images/rstudio_terminal.png -------------------------------------------------------------------------------- /images/view_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/images/view_run.png -------------------------------------------------------------------------------- /images/view_run_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/images/view_run_output.png -------------------------------------------------------------------------------- /inst/examples/mnist_mlp/mnist_mlp.R: -------------------------------------------------------------------------------- 1 | #' Trains a simple deep NN on the MNIST dataset. 2 | #' 3 | #' Gets to 98.40% test accuracy after 20 epochs (there is *a lot* of margin for 4 | #' parameter tuning). 5 | #' 6 | 7 | library(keras) 8 | 9 | # Hyperparameter flags --------------------------------------------------- 10 | 11 | FLAGS <- flags( 12 | flag_numeric("dropout1", 0.4), 13 | flag_numeric("dropout2", 0.3) 14 | ) 15 | 16 | # Data Preparation --------------------------------------------------- 17 | 18 | # The data, shuffled and split between train and test sets 19 | mnist <- dataset_mnist() 20 | x_train <- mnist$train$x 21 | y_train <- mnist$train$y 22 | x_test <- mnist$test$x 23 | y_test <- mnist$test$y 24 | 25 | # Reshape 26 | dim(x_train) <- c(nrow(x_train), 784) 27 | dim(x_test) <- c(nrow(x_test), 784) 28 | 29 | # Transform RGB values into [0,1] range 30 | x_train <- x_train / 255 31 | x_test <- x_test / 255 32 | 33 | # Convert class vectors to binary class matrices 34 | y_train <- to_categorical(y_train, 10) 35 | y_test <- to_categorical(y_test, 10) 36 | 37 | # Define Model -------------------------------------------------------------- 38 | 39 | model <- keras_model_sequential() 40 | model %>% 41 | layer_dense(units = 256, activation = 'relu', input_shape = c(784)) %>% 42 | layer_dropout(rate = FLAGS$dropout1) %>% 43 | layer_dense(units = 128, activation = 'relu') %>% 44 | layer_dropout(rate = FLAGS$dropout2) %>% 45 | layer_dense(units = 10, activation = 'softmax') 46 | 47 | model %>% compile( 48 | loss = 'categorical_crossentropy', 49 | optimizer = optimizer_rmsprop(lr = 0.001), 50 | metrics = c('accuracy') 51 | ) 52 | 53 | # Training & Evaluation ---------------------------------------------------- 54 | 55 | history <- model %>% fit( 56 | x_train, y_train, 57 | batch_size = 128, 58 | epochs = 20, 59 | verbose = 1, 60 | validation_split = 0.2 61 | ) 62 | 63 | plot(history) 64 | 65 | score <- model %>% evaluate( 66 | x_test, y_test, 67 | verbose = 0 68 | ) 69 | 70 | cat('Test loss:', score$loss, '\n') 71 | cat('Test accuracy:', score$acc, '\n') 72 | 73 | -------------------------------------------------------------------------------- /inst/rstudio/addins.dcf: -------------------------------------------------------------------------------- 1 | Name: Training Run 2 | Description: Execute a training run with the current source document 3 | Binding: add_in_training_run 4 | Interactive: false 5 | 6 | Name: View Latest Run 7 | Description: View the most recent training run 8 | Binding: add_in_view_latest_run 9 | Interactive: false 10 | 11 | Name: View Run History 12 | Description: View all training runs 13 | Binding: add_in_view_run_history 14 | Interactive: false 15 | -------------------------------------------------------------------------------- /inst/views/compare_runs.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Compare Runs 6 | 7 | 8 | 9 | 10 | 11 | {{{ jquery }}} 12 | 13 | {{{ materialize }}} 14 | 15 | {{{ material_icons }}} 16 | 17 | {{{ vue_js }}} 18 | 19 | {{{ c3 }}} 20 | 21 | {{{ metrics_charts }}} 22 | 23 | {{{ highlight_js }}} 24 | 25 | {{{ diff2html }}} 26 | 27 | {{{ dashboard }}} 28 | 29 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 49 | ### run_a.run_dir ###   ### run_b.run_dir ### 50 | 51 | 52 |
53 | 54 | 55 | 56 | 60 | 61 | 62 | 66 | 67 | 68 | 72 | 73 | 74 | 75 | 76 | 80 | 81 | 82 | 83 | 84 | 85 | 86 |
87 | 92 |
93 |
94 | 99 |
100 | 101 | 102 | 103 | 104 | 105 | 106 |
107 | 112 |
113 |
114 | 119 |
120 | 121 | 122 | 123 | 124 | 128 | 129 | 130 | 131 |
132 | 133 |
134 | 135 | 136 | 137 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | -------------------------------------------------------------------------------- /inst/views/components/dashboard.html: -------------------------------------------------------------------------------- 1 | 2 | 248 | 249 | 250 | 251 | 252 | 253 | 275 | 295 | 296 | 304 | 311 | 312 | 325 | 332 | 333 | 334 | 337 | 344 | 345 | 346 | 357 | 372 | 373 | 378 | 385 | 386 | 390 | 408 | 409 | 410 | 421 | 428 | 429 | 443 | 450 | 451 | 463 | 470 | 471 | 472 | 486 | 493 | 494 | 495 | 496 | 497 | 498 | 499 | -------------------------------------------------------------------------------- /inst/views/components/highlight_js.html: -------------------------------------------------------------------------------- 1 | 2 | 69 | 70 | 75 | 76 | 77 | 78 | 99 | 100 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /inst/views/components/jquery-AUTHORS.txt: -------------------------------------------------------------------------------- 1 | Authors ordered by first contribution. 2 | 3 | John Resig 4 | Gilles van den Hoven 5 | Michael Geary 6 | Stefan Petre 7 | Yehuda Katz 8 | Corey Jewett 9 | Klaus Hartl 10 | Franck Marcia 11 | Jörn Zaefferer 12 | Paul Bakaus 13 | Brandon Aaron 14 | Mike Alsup 15 | Dave Methvin 16 | Ed Engelhardt 17 | Sean Catchpole 18 | Paul Mclanahan 19 | David Serduke 20 | Richard D. Worth 21 | Scott González 22 | Ariel Flesler 23 | Jon Evans 24 | TJ Holowaychuk 25 | Michael Bensoussan 26 | Robert Katić 27 | Louis-Rémi Babé 28 | Earle Castledine 29 | Damian Janowski 30 | Rich Dougherty 31 | Kim Dalsgaard 32 | Andrea Giammarchi 33 | Mark Gibson 34 | Karl Swedberg 35 | Justin Meyer 36 | Ben Alman 37 | James Padolsey 38 | David Petersen 39 | Batiste Bieler 40 | Alexander Farkas 41 | Rick Waldron 42 | Filipe Fortes 43 | Neeraj Singh 44 | Paul Irish 45 | Iraê Carvalho 46 | Matt Curry 47 | Michael Monteleone 48 | Noah Sloan 49 | Tom Viner 50 | Douglas Neiner 51 | Adam J. Sontag 52 | Dave Reed 53 | Ralph Whitbeck 54 | Carl Fürstenberg 55 | Jacob Wright 56 | J. Ryan Stinnett 57 | unknown 58 | temp01 59 | Heungsub Lee 60 | Colin Snover 61 | Ryan W Tenney 62 | Pinhook 63 | Ron Otten 64 | Jephte Clain 65 | Anton Matzneller 66 | Alex Sexton 67 | Dan Heberden 68 | Henri Wiechers 69 | Russell Holbrook 70 | Julian Aubourg 71 | Gianni Alessandro Chiappetta 72 | Scott Jehl 73 | James Burke 74 | Jonas Pfenniger 75 | Xavi Ramirez 76 | Jared Grippe 77 | Sylvester Keil 78 | Brandon Sterne 79 | Mathias Bynens 80 | Timmy Willison 81 | Corey Frang 82 | Digitalxero 83 | Anton Kovalyov 84 | David Murdoch 85 | Josh Varner 86 | Charles McNulty 87 | Jordan Boesch 88 | Jess Thrysoee 89 | Michael Murray 90 | Lee Carpenter 91 | Alexis Abril 92 | Rob Morgan 93 | John Firebaugh 94 | Sam Bisbee 95 | Gilmore Davidson 96 | Brian Brennan 97 | Xavier Montillet 98 | Daniel Pihlstrom 99 | Sahab Yazdani 100 | avaly 101 | Scott Hughes 102 | Mike Sherov 103 | Greg Hazel 104 | Schalk Neethling 105 | Denis Knauf 106 | Timo Tijhof 107 | Steen Nielsen 108 | Anton Ryzhov 109 | Shi Chuan 110 | Berker Peksag 111 | Toby Brain 112 | Matt Mueller 113 | Justin 114 | Daniel Herman 115 | Oleg Gaidarenko 116 | Richard Gibson 117 | Rafaël Blais Masson 118 | cmc3cn <59194618@qq.com> 119 | Joe Presbrey 120 | Sindre Sorhus 121 | Arne de Bree 122 | Vladislav Zarakovsky 123 | Andrew E Monat 124 | Oskari 125 | Joao Henrique de Andrade Bruni 126 | tsinha 127 | Matt Farmer 128 | Trey Hunner 129 | Jason Moon 130 | Jeffery To 131 | Kris Borchers 132 | Vladimir Zhuravlev 133 | Jacob Thornton 134 | Chad Killingsworth 135 | Nowres Rafid 136 | David Benjamin 137 | Uri Gilad 138 | Chris Faulkner 139 | Elijah Manor 140 | Daniel Chatfield 141 | Nikita Govorov 142 | Wesley Walser 143 | Mike Pennisi 144 | Markus Staab 145 | Dave Riddle 146 | Callum Macrae 147 | Benjamin Truyman 148 | James Huston 149 | Erick Ruiz de Chávez 150 | David Bonner 151 | Akintayo Akinwunmi 152 | MORGAN 153 | Ismail Khair 154 | Carl Danley 155 | Mike Petrovich 156 | Greg Lavallee 157 | Daniel Gálvez 158 | Sai Lung Wong 159 | Tom H Fuertes 160 | Roland Eckl 161 | Jay Merrifield 162 | Allen J Schmidt Jr 163 | Jonathan Sampson 164 | Marcel Greter 165 | Matthias Jäggli 166 | David Fox 167 | Yiming He 168 | Devin Cooper 169 | Paul Ramos 170 | Rod Vagg 171 | Bennett Sorbo 172 | Sebastian Burkhard 173 | nanto 174 | Danil Somsikov 175 | Ryunosuke SATO 176 | Jean Boussier 177 | Adam Coulombe 178 | Andrew Plummer 179 | Mark Raddatz 180 | Dmitry Gusev 181 | Michał Gołębiowski 182 | Nguyen Phuc Lam 183 | Tom H Fuertes 184 | Brandon Johnson 185 | Jason Bedard 186 | Kyle Robinson Young 187 | Renato Oliveira dos Santos 188 | Chris Talkington 189 | Eddie Monge 190 | Terry Jones 191 | Jason Merino 192 | Jeremy Dunck 193 | Chris Price 194 | Amey Sakhadeo 195 | Anthony Ryan 196 | Dominik D. Geyer 197 | George Kats 198 | Lihan Li 199 | Ronny Springer 200 | Marian Sollmann 201 | Corey Frang 202 | Chris Antaki 203 | Noah Hamann 204 | David Hong 205 | Jakob Stoeck 206 | Christopher Jones 207 | Forbes Lindesay 208 | John Paul 209 | S. Andrew Sheppard 210 | Leonardo Balter 211 | Roman Reiß 212 | Benjy Cui 213 | Rodrigo Rosenfeld Rosas 214 | John Hoven 215 | Christian Kosmowski 216 | Liang Peng 217 | TJ VanToll 218 | -------------------------------------------------------------------------------- /inst/views/components/metrics_charts.html: -------------------------------------------------------------------------------- 1 | 337 | -------------------------------------------------------------------------------- /inst/views/metrics.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | {{{ jquery }}} 10 | 11 | {{{c3}}} 12 | 13 | {{{metrics_charts}}} 14 | 15 | 16 | 17 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /inst/views/view_run.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Training Run 6 | 7 | 8 | 9 | 10 | 11 | {{{ jquery }}} 12 | 13 | {{{ materialize }}} 14 | 15 | {{{ material_icons }}} 16 | 17 | {{{ vue_js }}} 18 | 19 | {{{ c3 }}} 20 | 21 | {{{ metrics_charts }}} 22 | 23 | {{{ highlight_js }}} 24 | 25 | {{{ dashboard }}} 26 | 27 | 34 | 35 | 36 | 37 | 38 | 39 |
40 | 41 | 42 | ### run_dir ### 43 | 44 | 45 |
46 | 47 |
48 | 49 |
50 | 51 |
52 | 53 | 59 | 60 | 66 | 67 | 73 | 74 | 80 | 81 | 87 | 88 | 94 | 95 | 96 | 102 | 103 |
104 | 105 |
106 | 107 | 129 | 130 | 149 | 150 | 151 |
152 | 153 |
154 | 155 |
156 | 157 | 181 | 182 |
183 | 184 | 185 |
186 | 187 |
188 | 189 |
190 | 191 | 192 | 193 | 194 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | -------------------------------------------------------------------------------- /man-roxygen/roxlate-metrics-format.R: -------------------------------------------------------------------------------- 1 | #' @md 2 | #' 3 | #' @section Metrics Data Frame: 4 | #' 5 | #' Metrics should be passed as a data frame with one column for each metric. 6 | #' If the metrics are not yet complete (e.g. only metrics for the 7 | #' first several epochs are provided) then metrics in yet to be completed 8 | #' epochs should use `NA` as their values. For example: 9 | #' 10 | #' ``` 11 | #' data.frame': 30 obs. of 4 variables: 12 | #' $ loss : num 0.423 0.201 NA NA NA ... 13 | #' $ acc : num 0.873 0.942 NA NA NA ... 14 | #' $ val_loss: num 0.174 0.121 NA NA NA ... 15 | #' $ val_acc : num 0.949 0.964 NA NA NA ... 16 | #' ``` 17 | #' 18 | #' If both metrics and validation metrics are provided, you should preface the 19 | #' name of the validation metric with `"val_"` (e.g. for a metric named `"loss"` 20 | #' provide validation metrics in `"val_loss"`). This indicates that the metrics 21 | #' are related which is useful e.g. when plotting metrics. 22 | #' 23 | -------------------------------------------------------------------------------- /man/as_run_dir.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/run_dir.R 3 | \name{as_run_dir} 4 | \alias{as_run_dir} 5 | \title{Extract run directory from an object} 6 | \usage{ 7 | as_run_dir(x) 8 | } 9 | \arguments{ 10 | \item{x}{Object to extract run directory from} 11 | } 12 | \value{ 13 | Run directory path(s) 14 | } 15 | \description{ 16 | Extract run directory from an object 17 | } 18 | \keyword{internal} 19 | -------------------------------------------------------------------------------- /man/clean_runs.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/clean_runs.R 3 | \name{clean_runs} 4 | \alias{clean_runs} 5 | \alias{purge_runs} 6 | \title{Clean run directories} 7 | \usage{ 8 | clean_runs( 9 | runs = ls_runs(runs_dir = runs_dir), 10 | runs_dir = getOption("tfruns.runs_dir", "runs"), 11 | confirm = interactive() 12 | ) 13 | 14 | purge_runs( 15 | runs_dir = getOption("tfruns.runs_dir", "runs"), 16 | confirm = interactive() 17 | ) 18 | } 19 | \arguments{ 20 | \item{runs}{Runs to clean. Can be specified as a data frame 21 | (as returned by \code{\link[=ls_runs]{ls_runs()}}) or as a character vector of 22 | run directories.} 23 | 24 | \item{runs_dir}{Directory containing runs. Defaults to "runs" beneath the 25 | current working directory (or to the value of the \code{tfruns.runs_dir} R 26 | option if specified).} 27 | 28 | \item{confirm}{\code{TRUE} to confirm before performing operation} 29 | } 30 | \description{ 31 | Remove run directories from the filesystem. 32 | } 33 | \details{ 34 | The \code{clean_runs()} function moves the specified runs (by default, 35 | all runs) into an "archive" subdirectory of the "runs" directory. 36 | 37 | The \code{purge_runs()} function permanently deletes the "archive" 38 | subdirectory. 39 | } 40 | \examples{ 41 | \dontrun{ 42 | clean_runs(ls_runs(completed == FALSE)) 43 | } 44 | 45 | } 46 | \seealso{ 47 | Other run management: 48 | \code{\link{copy_run}()} 49 | } 50 | \concept{run management} 51 | -------------------------------------------------------------------------------- /man/compare_runs.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/training_run.R 3 | \name{compare_runs} 4 | \alias{compare_runs} 5 | \title{Compare training runs} 6 | \usage{ 7 | compare_runs(runs = ls_runs(latest_n = 2), viewer = getOption("tfruns.viewer")) 8 | } 9 | \arguments{ 10 | \item{runs}{Character vector of 2 training run directories or 11 | data frame returned from \code{\link[=ls_runs]{ls_runs()}} with at least 2 elements.} 12 | 13 | \item{viewer}{Viewer to display training run information within 14 | (default to an internal page viewer if available, otherwise 15 | to the R session default web browser).} 16 | } 17 | \description{ 18 | Render a visual comparison of two training runs. The runs are 19 | displayed with the most recent run on the right and the 20 | earlier run on the left. 21 | } 22 | -------------------------------------------------------------------------------- /man/copy_run.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/copy.R 3 | \name{copy_run} 4 | \alias{copy_run} 5 | \alias{copy_run_files} 6 | \title{Copy run directories} 7 | \usage{ 8 | copy_run(run_dir, to = ".", rename = NULL) 9 | 10 | copy_run_files(run_dir, to = ".", rename = NULL) 11 | } 12 | \arguments{ 13 | \item{run_dir}{Training run directory or data frame returned from 14 | \code{\link[=ls_runs]{ls_runs()}}.} 15 | 16 | \item{to}{Name of parent directory to copy run(s) into. Defaults to the 17 | current working directory.} 18 | 19 | \item{rename}{Rename run directory after copying. If not specified this 20 | defaults to the basename of the run directory (e.g. 21 | "2017-09-24T10-54-00Z").} 22 | } 23 | \value{ 24 | Logical vector indicating which operation succeeded for each of the 25 | run directories specified. 26 | } 27 | \description{ 28 | Functions for exporting/copying run directories and run artifact files. 29 | } 30 | \details{ 31 | Use \code{copy_run} to copy one or more run directories. 32 | 33 | Use \code{copy_run_files} to copy only files saved/generated by training run 34 | scripts (e.g. saved models, checkpoints, etc.). 35 | } 36 | \examples{ 37 | \dontrun{ 38 | 39 | # export a run directory to the current working directory 40 | copy_run("runs/2017-09-24T10-54-00Z") 41 | 42 | # export to the current working directory then rename 43 | copy_run("runs/2017-09-24T10-54-00Z", rename = "best-run") 44 | 45 | # export artifact files only to the current working directory then rename 46 | copy_run_files("runs/2017-09-24T10-54-00Z", rename = "best-model") 47 | 48 | # export 3 best eval_acc to a "best-runs" directory 49 | copy_run(ls_runs(order = eval_acc)[1:3,], to = "best-runs") 50 | 51 | } 52 | } 53 | \seealso{ 54 | Other run management: 55 | \code{\link{clean_runs}()} 56 | } 57 | \concept{run management} 58 | -------------------------------------------------------------------------------- /man/flags.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flags.R 3 | \name{flags} 4 | \alias{flags} 5 | \alias{flag_numeric} 6 | \alias{flag_integer} 7 | \alias{flag_boolean} 8 | \alias{flag_string} 9 | \title{Flags for a training run} 10 | \usage{ 11 | flags( 12 | ..., 13 | config = Sys.getenv("R_CONFIG_ACTIVE", unset = "default"), 14 | file = "flags.yml", 15 | arguments = commandArgs(TRUE) 16 | ) 17 | 18 | flag_numeric(name, default, description = NULL) 19 | 20 | flag_integer(name, default, description = NULL) 21 | 22 | flag_boolean(name, default, description = NULL) 23 | 24 | flag_string(name, default, description = NULL) 25 | } 26 | \arguments{ 27 | \item{...}{One or more flag definitions} 28 | 29 | \item{config}{The configuration to use. Defaults to the active configuration 30 | for the current environment (as specified by the \code{R_CONFIG_ACTIVE} 31 | environment variable), or \code{default} when unset.} 32 | 33 | \item{file}{The flags YAML file to read} 34 | 35 | \item{arguments}{The command line arguments (as a character vector) to be 36 | parsed.} 37 | 38 | \item{name}{Flag name} 39 | 40 | \item{default}{Flag default value} 41 | 42 | \item{description}{Flag description} 43 | } 44 | \value{ 45 | Named list of training flags 46 | } 47 | \description{ 48 | Define the flags (name, type, default value, description) which paramaterize 49 | a training run. Optionally read overrides of the default values from a 50 | "flags.yml" config file and/or command line arguments. 51 | } 52 | \section{Config File Flags}{ 53 | 54 | 55 | Config file flags are defined a YAML configuration file (by default 56 | named "flags.yml"). Flags can either appear at the top-level of 57 | the YAML or can be inclued in named configuration sections 58 | (see the \href{https://github.com/rstudio/config}{config package} for 59 | details). 60 | } 61 | 62 | \section{Command Line Flags}{ 63 | 64 | 65 | Command line flags should be of the form \code{--key=value} or 66 | \verb{--key value}. The values are assumed to be valid \code{yaml} and 67 | will be converted using \code{\link[=yaml.load]{yaml.load()}}. 68 | } 69 | 70 | \examples{ 71 | \dontrun{ 72 | library(tfruns) 73 | 74 | # define flags and parse flag values from flags.yml and the command line 75 | FLAGS <- flags( 76 | flag_numeric('learning_rate', 0.01, 'Initial learning rate.'), 77 | flag_integer('max_steps', 5000, 'Number of steps to run trainer.'), 78 | flag_string('data_dir', 'MNIST-data', 'Directory for training data'), 79 | flag_boolean('fake_data', FALSE, 'If true, use fake data for testing') 80 | ) 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /man/is_run_active.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/run_dir.R 3 | \name{is_run_active} 4 | \alias{is_run_active} 5 | \title{Check for an active training run} 6 | \usage{ 7 | is_run_active() 8 | } 9 | \value{ 10 | \code{TRUE} if a training tun is currently active 11 | } 12 | \description{ 13 | Check for an active training run 14 | } 15 | -------------------------------------------------------------------------------- /man/latest_run.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/ls_runs.R 3 | \name{latest_run} 4 | \alias{latest_run} 5 | \title{Latest training run} 6 | \usage{ 7 | latest_run(runs_dir = getOption("tfruns.runs_dir", "runs")) 8 | } 9 | \arguments{ 10 | \item{runs_dir}{Directory containing runs. Defaults to "runs" beneath the 11 | current working directory (or to the value of the \code{tfruns.runs_dir} R 12 | option if specified).} 13 | } 14 | \value{ 15 | Named list with run attributes (or \code{NULL} if no runs found) 16 | } 17 | \description{ 18 | Latest training run 19 | } 20 | -------------------------------------------------------------------------------- /man/ls_runs.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/ls_runs.R 3 | \name{ls_runs} 4 | \alias{ls_runs} 5 | \title{List or view training runs} 6 | \usage{ 7 | ls_runs( 8 | subset = NULL, 9 | order = "start", 10 | decreasing = TRUE, 11 | latest_n = NULL, 12 | runs_dir = getOption("tfruns.runs_dir", "runs") 13 | ) 14 | } 15 | \arguments{ 16 | \item{subset}{Logical expression indicating rows to keep (missing values are 17 | taken as false). See \code{\link[=subset]{subset()}}.} 18 | 19 | \item{order}{Columns to order by (defaults to run start time)} 20 | 21 | \item{decreasing}{\code{TRUE} to use decreasing order (e.g. list most recent runs 22 | first)} 23 | 24 | \item{latest_n}{Limit query to the \code{latest_n} most recent runs} 25 | 26 | \item{runs_dir}{Directory containing runs. Defaults to "runs" beneath the 27 | current working directory (or to the value of the \code{tfruns.runs_dir} R 28 | option if specified).} 29 | } 30 | \value{ 31 | Data frame with training runs 32 | } 33 | \description{ 34 | List or view training runs 35 | } 36 | \details{ 37 | When printing the results of \code{ls_runs()}, only \code{run_dir}, 38 | \code{metric_loss}, \code{metric_val_loss}, and any columns specified in \code{order} will 39 | be printed. 40 | 41 | To view all fields, use \code{View(ls_runs())}. 42 | } 43 | -------------------------------------------------------------------------------- /man/reexports.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/reexports.R 3 | \docType{import} 4 | \name{reexports} 5 | \alias{reexports} 6 | \alias{\%>\%} 7 | \title{Objects exported from other packages} 8 | \keyword{internal} 9 | \description{ 10 | These objects are imported from other packages. Follow the links 11 | below to see their documentation. 12 | 13 | \describe{ 14 | \item{magrittr}{\code{\link[magrittr:pipe]{\%>\%}}} 15 | }} 16 | 17 | -------------------------------------------------------------------------------- /man/run_dir.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/run_dir.R 3 | \name{run_dir} 4 | \alias{run_dir} 5 | \title{Current run directory} 6 | \usage{ 7 | run_dir() 8 | } 9 | \value{ 10 | Active run direcotry (or current working directory as a fallback) 11 | } 12 | \description{ 13 | Returns the current training run directory. If a training run is 14 | not currently active (see \code{\link[=is_run_active]{is_run_active()}}) then the current 15 | working directory is returned. 16 | } 17 | -------------------------------------------------------------------------------- /man/run_info.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/ls_runs.R 3 | \name{run_info} 4 | \alias{run_info} 5 | \title{Summary of training run} 6 | \usage{ 7 | run_info(run_dir) 8 | } 9 | \arguments{ 10 | \item{run_dir}{Training run directory or data frame returned from 11 | \code{\link[=ls_runs]{ls_runs()}}.} 12 | } 13 | \value{ 14 | Training run summary object with timing, flags, model info, training 15 | and evaluation metrics, etc. If more than one \code{run_dir} is passed then 16 | a list of training run summary objects is returned. 17 | } 18 | \description{ 19 | Summary of training run 20 | } 21 | \seealso{ 22 | \code{\link[=view_run]{view_run()}} 23 | } 24 | -------------------------------------------------------------------------------- /man/save_run_comparison.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/training_run.R 3 | \name{save_run_comparison} 4 | \alias{save_run_comparison} 5 | \title{Save a run comparison as HTML} 6 | \usage{ 7 | save_run_comparison(runs = ls_runs(latest_n = 2), filename = "auto") 8 | } 9 | \arguments{ 10 | \item{runs}{Character vector of 2 training run directories or 11 | data frame returned from \code{\link[=ls_runs]{ls_runs()}} with at least 2 elements.} 12 | 13 | \item{filename}{Path to save the HTML to. If no \code{filename} is specified 14 | then a temporary file is used (the path to the file is returned invisibly).} 15 | } 16 | \description{ 17 | Save a run comparison as HTML 18 | } 19 | -------------------------------------------------------------------------------- /man/save_run_view.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/training_run.R 3 | \name{save_run_view} 4 | \alias{save_run_view} 5 | \title{Save a run view as HTML} 6 | \usage{ 7 | save_run_view(run_dir = latest_run(), filename = "auto") 8 | } 9 | \arguments{ 10 | \item{run_dir}{Training run directory or data frame returned from 11 | \code{\link[=ls_runs]{ls_runs()}}.} 12 | 13 | \item{filename}{Path to save the HTML to. If no \code{filename} is specified 14 | then a temporary file is used (the path to the file is returned invisibly).} 15 | } 16 | \description{ 17 | The saved view includes summary information (flags, metrics, model 18 | attributes, etc.), plot and console output, and the code used for the run. 19 | } 20 | \seealso{ 21 | \code{\link[=ls_runs]{ls_runs()}}, \code{\link[=run_info]{run_info()}}, \code{\link[=view_run]{view_run()}} 22 | } 23 | -------------------------------------------------------------------------------- /man/training_run.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/training_run.R 3 | \name{training_run} 4 | \alias{training_run} 5 | \title{Run a training script} 6 | \usage{ 7 | training_run( 8 | file = "train.R", 9 | context = "local", 10 | config = Sys.getenv("R_CONFIG_ACTIVE", unset = "default"), 11 | flags = NULL, 12 | properties = NULL, 13 | run_dir = NULL, 14 | artifacts_dir = getwd(), 15 | echo = TRUE, 16 | view = "auto", 17 | envir = parent.frame(), 18 | encoding = getOption("encoding") 19 | ) 20 | } 21 | \arguments{ 22 | \item{file}{Path to training script (defaults to "train.R")} 23 | 24 | \item{context}{Run context (defaults to "local")} 25 | 26 | \item{config}{The configuration to use. Defaults to the active configuration 27 | for the current environment (as specified by the \code{R_CONFIG_ACTIVE} 28 | environment variable), or \code{default} when unset.} 29 | 30 | \item{flags}{Named list with flag values (see \code{\link[=flags]{flags()}}) or path 31 | to YAML file containing flag values.} 32 | 33 | \item{properties}{Named character vector with run properties. Properties are 34 | additional metadata about the run which will be subsequently available via 35 | \code{\link[=ls_runs]{ls_runs()}}.} 36 | 37 | \item{run_dir}{Directory to store run data within} 38 | 39 | \item{artifacts_dir}{Directory to capture created and modified files within. 40 | Pass \code{NULL} to not capture any artifcats.} 41 | 42 | \item{echo}{Print expressions within training script} 43 | 44 | \item{view}{View the results of the run after training. The default "auto" 45 | will view the run when executing a top-level (printed) statement in an 46 | interactive session. Pass \code{TRUE} or \code{FALSE} to control whether the view is 47 | shown explictly. You can also pass "save" to save a copy of the 48 | run report at \code{tfruns.d/view.html}} 49 | 50 | \item{envir}{The environment in which the script should be evaluated} 51 | 52 | \item{encoding}{The encoding of the training script; see \code{\link[=file]{file()}}.} 53 | } 54 | \value{ 55 | Single row data frame with run flags, metrics, etc. 56 | } 57 | \description{ 58 | Run a training script 59 | } 60 | \details{ 61 | The training run will by default use a unique new run directory 62 | within the "runs" sub-directory of the current working directory (or to the 63 | value of the \code{tfruns.runs_dir} R option if specified). 64 | 65 | The directory name will be a timestamp (in GMT time). If a duplicate name is 66 | generated then the function will wait long enough to return a unique one. 67 | 68 | If you want to use an alternate directory to store run data you can either 69 | set the global \code{tfruns.runs_dir} R option, or you can pass a \code{run_dir} 70 | explicitly to \code{training_run()}, optionally using the \code{\link[=unique_run_dir]{unique_run_dir()}} 71 | function to generate a timestamp-based directory name. 72 | } 73 | -------------------------------------------------------------------------------- /man/tuning_run.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/training_run.R 3 | \name{tuning_run} 4 | \alias{tuning_run} 5 | \title{Tune hyperparameters using training flags} 6 | \usage{ 7 | tuning_run( 8 | file = "train.R", 9 | context = "local", 10 | config = Sys.getenv("R_CONFIG_ACTIVE", unset = "default"), 11 | flags = NULL, 12 | sample = NULL, 13 | properties = NULL, 14 | runs_dir = getOption("tfruns.runs_dir", "runs"), 15 | artifacts_dir = getwd(), 16 | echo = TRUE, 17 | confirm = interactive(), 18 | envir = parent.frame(), 19 | encoding = getOption("encoding") 20 | ) 21 | } 22 | \arguments{ 23 | \item{file}{Path to training script (defaults to "train.R")} 24 | 25 | \item{context}{Run context (defaults to "local")} 26 | 27 | \item{config}{The configuration to use. Defaults to the active configuration 28 | for the current environment (as specified by the \code{R_CONFIG_ACTIVE} 29 | environment variable), or \code{default} when unset.} 30 | 31 | \item{flags}{Either a named list with flag values (multiple values can be 32 | provided for each flag) or a data frame that contains pre-generated 33 | combinations of flags (e.g. via \code{\link[base:expand.grid]{base::expand.grid()}}). The latter can 34 | be useful for subsetting combinations. See 'Examples'.} 35 | 36 | \item{sample}{Sampling rate for flag combinations (defaults to 37 | running all combinations).} 38 | 39 | \item{properties}{Named character vector with run properties. Properties are 40 | additional metadata about the run which will be subsequently available via 41 | \code{\link[=ls_runs]{ls_runs()}}.} 42 | 43 | \item{runs_dir}{Directory containing runs. Defaults to "runs" beneath the 44 | current working directory (or to the value of the \code{tfruns.runs_dir} R 45 | option if specified).} 46 | 47 | \item{artifacts_dir}{Directory to capture created and modified files within. 48 | Pass \code{NULL} to not capture any artifcats.} 49 | 50 | \item{echo}{Print expressions within training script} 51 | 52 | \item{confirm}{Confirm before executing tuning run.} 53 | 54 | \item{envir}{The environment in which the script should be evaluated} 55 | 56 | \item{encoding}{The encoding of the training script; see \code{\link[=file]{file()}}.} 57 | } 58 | \value{ 59 | Data frame with summary of all training runs performed 60 | during tuning. 61 | } 62 | \description{ 63 | Run all combinations of the specifed training flags. The number of 64 | combinations can be reduced by specifying the \code{sample} parameter, which 65 | will result in a random sample of the flag combinations being run. 66 | } 67 | \examples{ 68 | \dontrun{ 69 | library(tfruns) 70 | 71 | # using a list as input to the flags argument 72 | runs <- tuning_run( 73 | system.file("examples/mnist_mlp/mnist_mlp.R", package = "tfruns"), 74 | flags = list( 75 | dropout1 = c(0.2, 0.3, 0.4), 76 | dropout2 = c(0.2, 0.3, 0.4) 77 | ) 78 | ) 79 | runs[order(runs$eval_acc, decreasing = TRUE), ] 80 | 81 | # using a data frame as input to the flags argument 82 | # resulting in the same combinations above, but remove those 83 | # where the combined dropout rate exceeds 1 84 | grid <- expand.grid( 85 | dropout1 = c(0.2, 0.3, 0.4), 86 | dropout2 = c(0.2, 0.3, 0.4) 87 | ) 88 | grid$combined_droput <- grid$dropout1 + grid$dropout2 89 | grid <- grid[grid$combined_droput <= 1, ] 90 | runs <- tuning_run( 91 | system.file("examples/mnist_mlp/mnist_mlp.R", package = "tfruns"), 92 | flags = grid[, c("dropout1", "dropout2")] 93 | ) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /man/unique_run_dir.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/run_dir.R 3 | \name{unique_run_dir} 4 | \alias{unique_run_dir} 5 | \title{Create a unique run directory} 6 | \usage{ 7 | unique_run_dir( 8 | runs_dir = getOption("tfruns.runs_dir", "runs"), 9 | seconds_scale = 0 10 | ) 11 | } 12 | \arguments{ 13 | \item{runs_dir}{Directory containing runs. Defaults to "runs" beneath the 14 | current working directory (or to the value of the \code{tfruns.runs_dir} R 15 | option if specified).} 16 | 17 | \item{seconds_scale}{Decimal scale for the seconds component of the 18 | timestamp. Defaults to 0 which results in only the rounded seconds value 19 | being used in the timestamp. Specify larger numbers to include a decimal 20 | component (useful if you need to create many unique run directories at the 21 | same time).} 22 | } 23 | \description{ 24 | Create a new uniquely named run directory within the specified \code{runs_dir}. 25 | } 26 | \details{ 27 | The directory name will be a timestamp (in GMT time). If a duplicate name is 28 | generated then the function will wait long enough to return a unique one. 29 | } 30 | -------------------------------------------------------------------------------- /man/view_run.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/training_run.R 3 | \name{view_run} 4 | \alias{view_run} 5 | \title{View a training run} 6 | \usage{ 7 | view_run(run_dir = latest_run(), viewer = getOption("tfruns.viewer")) 8 | } 9 | \arguments{ 10 | \item{run_dir}{Training run directory or data frame returned from 11 | \code{\link[=ls_runs]{ls_runs()}}.} 12 | 13 | \item{viewer}{Viewer to display training run information within 14 | (default to an internal page viewer if available, otherwise 15 | to the R session default web browser).} 16 | } 17 | \description{ 18 | View metrics and other attributes of a training run. 19 | } 20 | \seealso{ 21 | \code{\link[=ls_runs]{ls_runs()}}, \code{\link[=run_info]{run_info()}} 22 | } 23 | -------------------------------------------------------------------------------- /man/view_run_metrics.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/metrics_viewer.R 3 | \name{view_run_metrics} 4 | \alias{view_run_metrics} 5 | \alias{update_run_metrics} 6 | \title{View metrics for a training run} 7 | \usage{ 8 | view_run_metrics(metrics) 9 | 10 | update_run_metrics(viewer, metrics) 11 | } 12 | \arguments{ 13 | \item{metrics}{Data frame containing run metrics} 14 | 15 | \item{viewer}{Viewer object returned from \code{view_run_metrics()}.} 16 | } 17 | \description{ 18 | Interactive D3 visualization of metrics for a training run. Metrics will 19 | be displayed in the RStudio Viewer (if available), otherwise will be 20 | displayed in an external web browser. 21 | } 22 | \note{ 23 | Metrics named \code{"acc"} or \code{"accuracy"} will automatically use \code{1.0} as the 24 | maximum value on their y-axis scale. 25 | } 26 | \section{Metrics Data Frame}{ 27 | 28 | 29 | Metrics should be passed as a data frame with one column for each metric. 30 | If the metrics are not yet complete (e.g. only metrics for the 31 | first several epochs are provided) then metrics in yet to be completed 32 | epochs should use \code{NA} as their values. For example: 33 | 34 | \if{html}{\out{
}}\preformatted{data.frame': 30 obs. of 4 variables: 35 | $ loss : num 0.423 0.201 NA NA NA ... 36 | $ acc : num 0.873 0.942 NA NA NA ... 37 | $ val_loss: num 0.174 0.121 NA NA NA ... 38 | $ val_acc : num 0.949 0.964 NA NA NA ... 39 | }\if{html}{\out{
}} 40 | 41 | If both metrics and validation metrics are provided, you should preface the 42 | name of the validation metric with \code{"val_"} (e.g. for a metric named \code{"loss"} 43 | provide validation metrics in \code{"val_loss"}). This indicates that the metrics 44 | are related which is useful e.g. when plotting metrics. 45 | } 46 | 47 | \section{Realtime Updates}{ 48 | 49 | 50 | Metrics can be updated in real-time by calling the \code{update_run_metrics()} 51 | with the run viewer instance returned from \code{view_run_metrics()}. For example: 52 | 53 | \if{html}{\out{
}}\preformatted{# view metrics 54 | viewer <- view_run_metrics(metrics) 55 | 56 | # update with new metrics 57 | update_run_metrics(viewer, updated_metrics) 58 | }\if{html}{\out{
}} 59 | } 60 | 61 | \seealso{ 62 | write_run_metrics 63 | } 64 | -------------------------------------------------------------------------------- /man/write_run_data.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/run_metadata.R 3 | \name{write_run_data} 4 | \alias{write_run_data} 5 | \title{Write run data (deprecated)} 6 | \usage{ 7 | write_run_data(type, data) 8 | } 9 | \arguments{ 10 | \item{type}{Type of metadata to write. Standard types include "flags", 11 | "sources", "properties", "metrics", and "evaluation". You can 12 | also specify a custom type (see \emph{Custom Types} section below).} 13 | 14 | \item{data}{Metadata to write: 15 | \itemize{ 16 | \item "flags" --- Named list of training flags 17 | \item "source" --- Directory to copy source files from 18 | \item "properties" --- Named list of arbitrary properties. Note 19 | that properties will be stored as strings. 20 | \item "metrics" --- Data frame with training run metrics 21 | (see \emph{Metrics Data Frame} below). 22 | \item "evaluation" --- Named list of evaluation metrics. 23 | \item "error" --- Named list with 'message' and 'traceback' 24 | \item "\" -- Function used to write the data 25 | (see \emph{Custom Types} section below). 26 | }} 27 | } 28 | \description{ 29 | Deprecated alias for \code{\link[=write_run_metadata]{write_run_metadata()}}. 30 | } 31 | \keyword{internal} 32 | -------------------------------------------------------------------------------- /man/write_run_metadata.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/run_metadata.R 3 | \name{write_run_metadata} 4 | \alias{write_run_metadata} 5 | \title{Write run metadata} 6 | \usage{ 7 | write_run_metadata(type, data, run_dir = NULL) 8 | } 9 | \arguments{ 10 | \item{type}{Type of metadata to write. Standard types include "flags", 11 | "sources", "properties", "metrics", and "evaluation". You can 12 | also specify a custom type (see \emph{Custom Types} section below).} 13 | 14 | \item{data}{Metadata to write: 15 | \itemize{ 16 | \item "flags" --- Named list of training flags 17 | \item "source" --- Directory to copy source files from 18 | \item "properties" --- Named list of arbitrary properties. Note 19 | that properties will be stored as strings. 20 | \item "metrics" --- Data frame with training run metrics 21 | (see \emph{Metrics Data Frame} below). 22 | \item "evaluation" --- Named list of evaluation metrics. 23 | \item "error" --- Named list with 'message' and 'traceback' 24 | \item "\" -- Function used to write the data 25 | (see \emph{Custom Types} section below). 26 | }} 27 | 28 | \item{run_dir}{Run directory to write metadata into (defaults 29 | to currently active run)} 30 | } 31 | \description{ 32 | Record various types of training run metadata This function can be called 33 | even when a run directory isn't active (metadata will only be written if 34 | and when a run directory is initialized). 35 | } 36 | \note{ 37 | \code{write_run_data()} is deprecated and is provided as an alias 38 | for backward compatibility. 39 | } 40 | \section{Metrics Data Frame}{ 41 | 42 | 43 | Metrics should be passed as a data frame with one column for each metric. 44 | If the metrics are not yet complete (e.g. only metrics for the 45 | first several epochs are provided) then metrics in yet to be completed 46 | epochs should use \code{NA} as their values. For example: 47 | 48 | \if{html}{\out{
}}\preformatted{data.frame': 30 obs. of 4 variables: 49 | $ loss : num 0.423 0.201 NA NA NA ... 50 | $ acc : num 0.873 0.942 NA NA NA ... 51 | $ val_loss: num 0.174 0.121 NA NA NA ... 52 | $ val_acc : num 0.949 0.964 NA NA NA ... 53 | }\if{html}{\out{
}} 54 | 55 | If both metrics and validation metrics are provided, you should preface the 56 | name of the validation metric with \code{"val_"} (e.g. for a metric named \code{"loss"} 57 | provide validation metrics in \code{"val_loss"}). This indicates that the metrics 58 | are related which is useful e.g. when plotting metrics. 59 | } 60 | 61 | \section{Custom Types}{ 62 | 63 | 64 | You can pass a type with an arbitary name along with a function that 65 | should be used to writes the data. The function will be passed a 66 | single \code{data_dir} argument. For example: 67 | 68 | \if{html}{\out{
}}\preformatted{write_run_metadata("images", function(data_dir) \{ 69 | # write into data_dir here 70 | \}) 71 | }\if{html}{\out{
}} 72 | } 73 | 74 | \keyword{internal} 75 | -------------------------------------------------------------------------------- /pkgdown/_pkgdown.yml: -------------------------------------------------------------------------------- 1 | template: 2 | params: 3 | bootswatch: cosmo 4 | 5 | navbar: 6 | title: "tfruns" 7 | type: inverse 8 | left: 9 | - text: "Home" 10 | href: index.html 11 | - text: "Reference" 12 | href: reference/index.html 13 | right: 14 | - icon: fa-github 15 | href: https://github.com/rstudio/tfruns 16 | 17 | reference: 18 | - title: "Training" 19 | contents: 20 | - training_run 21 | - tuning_run 22 | - flags 23 | - run_dir 24 | - unique_run_dir 25 | 26 | - title: "Visualizing Runs" 27 | contents: 28 | - view_run 29 | - compare_runs 30 | - save_run_view 31 | - save_run_comparison 32 | 33 | - title: "Managing Runs" 34 | contents: 35 | - ls_runs 36 | - latest_run 37 | - run_info 38 | - clean_runs 39 | - purge_runs 40 | - copy_run 41 | - copy_run_files 42 | 43 | - title: "Package API" 44 | description: "Functions for use by R packages that integrate with tfruns" 45 | contents: 46 | - is_run_active 47 | - as_run_dir 48 | - view_run_metrics 49 | - update_run_metrics 50 | - write_run_metadata 51 | 52 | -------------------------------------------------------------------------------- /pkgdown/extra.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/pkgdown/extra.css -------------------------------------------------------------------------------- /pkgdown/extra.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/pkgdown/extra.js -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | library(tfruns) 3 | 4 | clean_runs(confirm = FALSE) 5 | purge_runs(confirm = FALSE) 6 | 7 | test_check("tfruns") 8 | -------------------------------------------------------------------------------- /tests/testthat/extra.dat: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | -------------------------------------------------------------------------------- /tests/testthat/flags-learning-rate.yml: -------------------------------------------------------------------------------- 1 | 2 | learning_rate: 0.05 3 | -------------------------------------------------------------------------------- /tests/testthat/flags-override.yml: -------------------------------------------------------------------------------- 1 | learning_rate: 0.02 2 | -------------------------------------------------------------------------------- /tests/testthat/flags-precision.R: -------------------------------------------------------------------------------- 1 | 2 | library(tfruns) 3 | 4 | FLAGS <- flags( 5 | flag_numeric('learning_rate', 2e-5, 'Initial learning rate.'), 6 | flag_numeric('max_steps', 1e-6, 'Number of steps to run trainer.') 7 | ) 8 | 9 | -------------------------------------------------------------------------------- /tests/testthat/flags-profile-override.yml: -------------------------------------------------------------------------------- 1 | myconfig: 2 | learning_rate: 0.03 3 | -------------------------------------------------------------------------------- /tests/testthat/flags.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/tests/testthat/flags.rds -------------------------------------------------------------------------------- /tests/testthat/helper.R: -------------------------------------------------------------------------------- 1 | with_tests_dir <- function(code){ 2 | test_dir <- testthat::test_path() 3 | withr::with_dir(test_dir, code) 4 | } 5 | 6 | 7 | define_flags <- function(...) { 8 | flags( 9 | flag_numeric('learning_rate', 0.01, 'Initial learning rate.'), 10 | flag_integer('max_steps', 5000, 'Number of steps to run trainer.'), 11 | flag_string('data_dir', 'MNIST-data', 'Directory for training data'), 12 | flag_boolean('fake_data', FALSE, 'If true, use fake data for testing'), 13 | ... 14 | ) 15 | } 16 | 17 | 18 | expect_success <- function(expr) { 19 | expect_error(force(expr), NA) 20 | } 21 | -------------------------------------------------------------------------------- /tests/testthat/metrics.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/tests/testthat/metrics.rds -------------------------------------------------------------------------------- /tests/testthat/subdir/extra.dat: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | -------------------------------------------------------------------------------- /tests/testthat/test-copy.R: -------------------------------------------------------------------------------- 1 | context("copy") 2 | 3 | 4 | # context 3 fake training runs (one of which that generates files) 5 | 6 | test_that("can create training runs",{ 7 | with_tests_dir({ 8 | lr <- training_run(view = FALSE, echo = FALSE) 9 | expect_is(lr, "data.frame") 10 | }) 11 | with_tests_dir({ 12 | training_run(view = FALSE, echo = FALSE) 13 | expect_is(lr, "data.frame") 14 | }) 15 | with_tests_dir({ 16 | training_run("write_run_data.R", view = FALSE, echo = FALSE) 17 | expect_is(lr, "data.frame") 18 | }) 19 | 20 | with_tests_dir({ 21 | lr <- latest_run() 22 | expect_is(lr, "tfruns_run") 23 | }) 24 | }) 25 | 26 | test_that("copy_run copies run directory", { 27 | with_tests_dir({ 28 | copy_run(latest_run(), rename = "copied-run-dir") 29 | expect_true(file.exists(file.path("copied-run-dir", "subdir", "extra.dat"))) 30 | expect_true(file.exists(file.path("copied-run-dir", "tfruns.d", "source.tar.gz"))) 31 | unlink("copied-run-dir", recursive = TRUE) 32 | }) 33 | }) 34 | 35 | test_that("copy_run_files copies run artifacts", { 36 | with_tests_dir({ 37 | copy_run_files(latest_run(), rename = "run-artifacts") 38 | expect_true(file.exists(file.path("run-artifacts", "extra.dat"))) 39 | unlink("run-artifacts", recursive = TRUE) 40 | }) 41 | }) 42 | 43 | test_that("copy_runs successfully copies run directories", { 44 | with_tests_dir({ 45 | last_3_runs <- ls_runs()[1:3,] 46 | copy_run(last_3_runs, "last-three") 47 | expect_true(all( 48 | utils::file_test( 49 | "-d", file.path("last-three", basename(as_run_dir(last_3_runs)))) 50 | )) 51 | unlink("last-three", recursive = TRUE) 52 | }) 53 | }) 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /tests/testthat/test-flags.R: -------------------------------------------------------------------------------- 1 | context("flags") 2 | 3 | test_that("flags can be defined", { 4 | with_tests_dir({ 5 | FLAGS <- define_flags() 6 | expect_equivalent(FLAGS, readRDS("flags.rds")) 7 | }) 8 | }) 9 | 10 | test_that("flags are assigned the correct types", { 11 | with_tests_dir({ 12 | FLAGS <- define_flags() 13 | expect_type(FLAGS$learning_rate, "double") 14 | expect_type(FLAGS$max_steps, "integer") 15 | expect_type(FLAGS$data_dir, "character") 16 | expect_type(FLAGS$fake_data, "logical") 17 | }) 18 | }) 19 | 20 | test_that("flags_parse returns defaults when there are no overrides", { 21 | with_tests_dir({ 22 | FLAGS <- define_flags() 23 | expect_equivalent(FLAGS, readRDS("flags.rds")) 24 | }) 25 | }) 26 | 27 | test_that("flags_parse overrides based on command line args", { 28 | with_tests_dir({ 29 | FLAGS <- define_flags(arguments = c("--learning-rate", "0.02")) 30 | expect_equal(FLAGS$learning_rate, 0.02) 31 | }) 32 | }) 33 | 34 | test_that("flags_parse throws an error for unknown command line args", { 35 | with_tests_dir({ 36 | expect_error({ 37 | FLAGS <- define_flags(arguments = c("--learn-rate", "0.02")) 38 | }) 39 | }) 40 | }) 41 | 42 | test_that("flags_parse overrides based on config file values", { 43 | with_tests_dir({ 44 | FLAGS <- define_flags(file = "flags-override.yml") 45 | expect_equal(FLAGS$learning_rate, 0.02) 46 | FLAGS <- define_flags(file = "flags-profile-override.yml", config = "myconfig") 47 | expect_equal(FLAGS$learning_rate, 0.03) 48 | }) 49 | }) 50 | 51 | test_that("flags_parse skips --args for passthrough args", { 52 | with_tests_dir({ 53 | FLAGS <- flags( 54 | flag_numeric("gradient_descent_optimizer", 0.5), 55 | arguments = list( 56 | "--gradient-descent-optimizer", 57 | "0.47", 58 | "--args", 59 | "--job-dir", 60 | "gs://rstudio-cloudml/mnist/staging/2") 61 | ) 62 | 63 | expect_equal(FLAGS$gradient_descent_optimizer, 0.47) 64 | }) 65 | }) 66 | -------------------------------------------------------------------------------- /tests/testthat/test-run-data.R: -------------------------------------------------------------------------------- 1 | context("run_data") 2 | 3 | 4 | run_dir <- with_tests_dir({ 5 | x <- training_run("write_run_data.R", echo = FALSE)$run_dir 6 | normalizePath(x, winslash = "/") 7 | }) 8 | 9 | 10 | run_data <- function(...) { 11 | with_tests_dir({ 12 | run_dir <- training_run("write_run_data.R", echo = FALSE)$run_dir 13 | run_dir <- normalizePath(run_dir, winslash = "/") 14 | file.path(run_dir, "tfruns.d", ...) 15 | }) 16 | } 17 | 18 | expect_run_data <- function(...) { 19 | expect_true(file.exists(run_data(...))) 20 | } 21 | 22 | test_that("flags are written to run_dir", { 23 | expect_run_data("flags.json") 24 | }) 25 | 26 | test_that("sources are written to run_dir", { 27 | expect_run_data("source.tar.gz") 28 | }) 29 | 30 | test_that("metrics are written to run_dir", { 31 | expect_run_data("metrics.json") 32 | }) 33 | 34 | test_that("properites are written to run_dir", { 35 | expect_run_data("properties", "foo") 36 | expect_run_data("properties", "index") 37 | expect_equal(readLines(file.path(run_dir, "tfruns.d", "properties", "foo")), "bar") 38 | }) 39 | 40 | test_that("custom run data can be written", { 41 | expect_run_data("foo") 42 | }) 43 | 44 | 45 | test_that("created and modified files are copied to the run_dir", { 46 | expect_true(file.exists(file.path(run_dir, 'extra.dat'))) 47 | expect_true(file.exists(file.path(run_dir, 'subdir', 'extra.dat'))) 48 | }) 49 | 50 | test_that("precision of metrics", { 51 | run_dir <- with_tests_dir({ 52 | x <- training_run("flags-precision.R", echo = FALSE)$run_dir 53 | normalizePath(x, winslash = "/") 54 | }) 55 | 56 | flags <- jsonlite::read_json( 57 | path = file.path(run_dir, "tfruns.d", "flags.json") 58 | ) 59 | 60 | expect_equal(flags$learning_rate, 2e-5) 61 | expect_equal(flags$max_steps, 1e-6) 62 | 63 | }) 64 | -------------------------------------------------------------------------------- /tests/testthat/test-runs.R: -------------------------------------------------------------------------------- 1 | context("runs") 2 | 3 | run_dir <- with_tests_dir({ 4 | x <- training_run(echo = FALSE)$run_dir 5 | normalizePath(x, winslash = "/") 6 | }) 7 | 8 | 9 | test_that("run dir is created by initialize_run()", { 10 | expect_true(dir.exists(run_dir)) 11 | }) 12 | 13 | test_that("list runs returns a data frame", { 14 | runs <- ls_runs() 15 | expect_true(is.data.frame(runs)) 16 | expect_true(all(c("start", "run_dir") %in% colnames(runs))) 17 | }) 18 | 19 | test_that("latest_run retrieves run_dir", { 20 | with_tests_dir({ 21 | expect_identical(basename(run_dir), basename(latest_run()$run_dir)) 22 | }) 23 | }) 24 | 25 | test_that("completed flag is set by training_run", { 26 | with_tests_dir({ 27 | expect_true(ls_runs(latest_n = 1)$completed) 28 | }) 29 | }) 30 | 31 | 32 | test_that("clean_runs and purge_runs remove runs", { 33 | # then clean 34 | with_tests_dir({ 35 | clean_runs(confirm = FALSE) 36 | expect_equal(nrow(ls_runs()), 0) 37 | expect_true(nrow(ls_runs(runs_dir = "runs/archive")) > 0) 38 | 39 | # then purge 40 | purge_runs(confirm = FALSE) 41 | expect_equal(nrow(ls_runs(runs_dir = "runs/archive")), 0) 42 | }) 43 | }) 44 | 45 | test_that("flags.yml can be passed to training_run", { 46 | with_tests_dir({ 47 | training_run(flags = "flags-learning-rate.yml", echo = FALSE) 48 | expect_equal(ls_runs(latest_n = 1)$flag_learning_rate, 0.05) 49 | }) 50 | }) 51 | 52 | test_that("training errors are handled", { 53 | tryCatch( 54 | { 55 | with_tests_dir({ 56 | training_run("train-error.R", echo = FALSE) 57 | }) 58 | }, error = function(e) { 59 | expect_equal(e$message, "Training error occurred") 60 | } 61 | ) 62 | 63 | run <- with_tests_dir({ 64 | ls_runs(latest_n = 1) 65 | }) 66 | 67 | expect_true(!run$completed) 68 | expect_equal(run$error_message, "Training error occurred") 69 | }) 70 | 71 | -------------------------------------------------------------------------------- /tests/testthat/test-tuning.R: -------------------------------------------------------------------------------- 1 | context("tuning") 2 | 3 | 4 | test_that("tuning_run throws graceful errors with wrong sample argument", { 5 | 6 | with_tests_dir({ 7 | expect_error( 8 | tuning_run("write_run_data.R", 9 | confirm = FALSE, 10 | flags = list( 11 | learning_rate = c(0.01, 0.02), 12 | max_steps = c(2500, 500) 13 | ), 14 | sample = 1.1, 15 | echo = FALSE 16 | ) 17 | ) 18 | }) 19 | 20 | with_tests_dir({ 21 | expect_error( 22 | tuning_run("write_run_data.R", 23 | confirm = FALSE, 24 | flags = list( 25 | learning_rate = c(0.01, 0.02), 26 | max_steps = c(2500, 500) 27 | ), 28 | sample = 0, 29 | echo = FALSE 30 | ) 31 | ) 32 | }) 33 | 34 | runs <- with_tests_dir({ 35 | tuning_run("write_run_data.R", 36 | confirm = FALSE, 37 | flags = list( 38 | learning_rate = c(0.01, 0.02), 39 | max_steps = c(2500, 500) 40 | ), 41 | sample = 1e-6, 42 | echo = FALSE 43 | ) 44 | }) 45 | expect_equal(nrow(runs), 1) 46 | 47 | }) 48 | 49 | test_that("tuning_run can execute multiple runs", { 50 | 51 | runs <- with_tests_dir({ 52 | tuning_run("write_run_data.R", 53 | confirm = FALSE, 54 | flags = list( 55 | learning_rate = c(0.01, 0.02), 56 | max_steps = c(2500, 500) 57 | ), 58 | sample = 1, 59 | echo = FALSE 60 | ) 61 | }) 62 | 63 | expect_equal(nrow(runs), 4) 64 | }) 65 | 66 | test_that("tuning_run can correctly handle different types of inputs for flags", { 67 | 68 | # specify flags as data frame 69 | grid <- expand.grid( 70 | learning_rate = c(0.01, 0.02), 71 | max_steps = c(2500, 500, 99) 72 | ) 73 | runs <- with_tests_dir(tuning_run("write_run_data.R", 74 | confirm = FALSE, flags = grid 75 | )) 76 | expect_equal(nrow(runs), 6) 77 | 78 | # specify none 79 | expect_error( 80 | with_tests_dir(tuning_run("write_run_data.R", confirm = FALSE), 81 | "flags must be specified as a named list" 82 | )) 83 | 84 | # supply unnamed flags 85 | expect_error( 86 | tuning_run("write_run_data.R", 87 | confirm = FALSE, 88 | flags = list(c(0.01, 0.02), c(2500, 500)) 89 | ), 90 | "as a named list" 91 | ) 92 | }) 93 | 94 | 95 | # tear down 96 | runs_dirs <- with_tests_dir(normalizePath(list.dirs("runs", recursive = FALSE))) 97 | unlink(runs_dirs, recursive = TRUE) 98 | -------------------------------------------------------------------------------- /tests/testthat/train-error.R: -------------------------------------------------------------------------------- 1 | 2 | # train script w/ error called by training_run in tests 3 | 4 | library(tfruns) 5 | 6 | FLAGS <- flags( 7 | flag_numeric("learning_rate", 0.01, "Learning rate") 8 | ) 9 | 10 | stop("Training error occurred") 11 | -------------------------------------------------------------------------------- /tests/testthat/train.R: -------------------------------------------------------------------------------- 1 | 2 | # train script called by training_run in tests 3 | 4 | library(tfruns) 5 | 6 | FLAGS <- flags( 7 | flag_numeric("learning_rate", 0.01, "Learning rate") 8 | ) 9 | -------------------------------------------------------------------------------- /tests/testthat/write_run_data.R: -------------------------------------------------------------------------------- 1 | 2 | library(tfruns) 3 | 4 | FLAGS <- flags( 5 | flag_numeric('learning_rate', 0.01, 'Initial learning rate.'), 6 | flag_integer('max_steps', 5000, 'Number of steps to run trainer.') 7 | ) 8 | 9 | metrics <- readRDS("metrics.rds") 10 | write_run_metadata("metrics", metrics) 11 | 12 | write_run_metadata("properties", list(foo = "bar", index = 42)) 13 | 14 | write_run_metadata("foo", function(data_dir) { 15 | file.create(file.path(data_dir, "foo")) 16 | }) 17 | 18 | writeLines(c("1", "2", "3"), 'extra.dat') 19 | writeLines(c("1", "2", "3"), file.path('subdir', 'extra.dat')) 20 | -------------------------------------------------------------------------------- /tfruns.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: No 4 | SaveWorkspace: No 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | 18 | BuildType: Package 19 | PackageUseDevtools: Yes 20 | PackageInstallArgs: --no-multiarch --with-keep.source 21 | PackageRoxygenize: rd,collate,namespace 22 | -------------------------------------------------------------------------------- /vignettes/images/compare_runs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/vignettes/images/compare_runs.png -------------------------------------------------------------------------------- /vignettes/images/ls_runs_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/vignettes/images/ls_runs_compare.png -------------------------------------------------------------------------------- /vignettes/images/ls_runs_rstudio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/vignettes/images/ls_runs_rstudio.png -------------------------------------------------------------------------------- /vignettes/images/rstudio_addin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/vignettes/images/rstudio_addin.png -------------------------------------------------------------------------------- /vignettes/images/rstudio_publish.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/vignettes/images/rstudio_publish.png -------------------------------------------------------------------------------- /vignettes/images/rstudio_terminal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/vignettes/images/rstudio_terminal.png -------------------------------------------------------------------------------- /vignettes/images/view_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/vignettes/images/view_run.png -------------------------------------------------------------------------------- /vignettes/images/view_run_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rstudio/tfruns/d6e8b6744235abf821644599165d1b77fa65f91a/vignettes/images/view_run_output.png -------------------------------------------------------------------------------- /vignettes/managing.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Managing Runs" 3 | output: 4 | rmarkdown::html_vignette: default 5 | vignette: > 6 | %\VignetteIndexEntry{Managing Runs} 7 | %\VignetteEngine{knitr::rmarkdown} 8 | %\VignetteEncoding{UTF-8} 9 | type: docs 10 | repo: https://github.com/rstudio/tfruns 11 | menu: 12 | main: 13 | name: "Managing Training Runs" 14 | identifier: "tools-tfruns-managing" 15 | parent: "tfruns-top" 16 | weight: 30 17 | --- 18 | 19 | ```{r setup, include=FALSE} 20 | knitr::opts_chunk$set(eval = FALSE) 21 | ``` 22 | 23 | ## Run Output 24 | 25 | Any graphical or console output as well as file artifacts created by a training run (e.g. saved models or saved model weights) can be viewed from the **Output** tab of the run view: 26 | 27 | 28 | 29 | You can use the `copy_run_files()` function to export file artifacts from runs into another directory. For example: 30 | 31 | ```{r} 32 | copy_run_files("runs/2017-09-24T10-54-00Z", to = "saved-model") 33 | ``` 34 | 35 | You can also use the `copy_run()` function to export a run directory in it's entirety. For example, this code exports the specified run to a "best-run" directory: 36 | 37 | ```{r} 38 | copy_run("runs/2017-09-24T10-54-00Z", to = "best-run") 39 | ``` 40 | 41 | Note that `copy_run()` will accept any number of runs. For example, this code exports all run directories with an evaluation accuracy greater than 0.98 to a "best-runs" directory: 42 | 43 | ```{r} 44 | copy_run(ls_runs(eval_acc >= 0.98), to = "best-runs") 45 | ``` 46 | 47 | ## Cleaning Runs 48 | 49 | You can use the `clean_runs()` function to archive a set of runs you no longer need the data from. For example, this code archives all runs with an eval accuracy less than 0.98: 50 | 51 | ```{r} 52 | clean_runs(ls_runs(eval_acc < 0.98)) 53 | ``` 54 | 55 | If you don't specify a set of runs to clean then all runs will be archived: 56 | 57 | ```{r} 58 | clean_runs() # archives all runs in the "runs" directory 59 | ``` 60 | 61 | Note that you'll always get a confirmation prompt before the runs are actually archived. 62 | 63 | ## Purging Runs 64 | 65 | When runs are archived they are moved to the "archive" subdirectory of the "runs" directory. If you want to permanently remove runs from the archive you call the `purge_runs()` function: 66 | 67 | ```{r} 68 | purge_runs() 69 | ``` 70 | 71 | 72 | -------------------------------------------------------------------------------- /vignettes/overview.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "tfruns: Track and Visualize Training Runs" 3 | output: 4 | rmarkdown::html_vignette: default 5 | vignette: > 6 | %\VignetteIndexEntry{Introduction to tfruns} 7 | %\VignetteEngine{knitr::rmarkdown} 8 | %\VignetteEncoding{UTF-8} 9 | type: docs 10 | repo: https://github.com/rstudio/tfruns 11 | menu: 12 | main: 13 | name: "Introduction to tfruns" 14 | identifier: "tools-tfruns-overview" 15 | parent: "tfruns-top" 16 | weight: 10 17 | aliases: 18 | - /tools/tfruns/ 19 | --- 20 | 21 | ```{r setup, include=FALSE} 22 | knitr::opts_chunk$set(eval = FALSE) 23 | ``` 24 | 25 | 26 | 27 | 28 | The **tfruns** package provides a suite of tools for tracking, visualizing, and managing TensorFlow training runs and experiments from R: 29 | 30 | - Track the hyperparameters, metrics, output, and source code of every training run. 31 | 32 | - Compare hyperparmaeters and metrics across runs to find the best performing model. 33 | 34 | - Automatically generate reports to visualize individual training runs or comparisons between runs. 35 | 36 | - No changes to source code required (run data is automatically captured for all Keras and TF Estimator models). 37 | 38 | ## Installation 39 | 40 | You can install the **tfruns** package from CRAN as follows: 41 | 42 | ```{r} 43 | install.packages("tfruns") 44 | ``` 45 | 46 | The package is intended to be used with the [keras](https://tensorflow.rstudio.com/keras/) package, which provides a higher level interfaces to TensorFlow from R. It can be installed with: 47 | 48 | ```{r} 49 | install.packages("keras") 50 | ``` 51 | 52 | ## Training 53 | 54 | In the following sections we'll describe the various capabilities of **tfruns**. Our example training script ([mnist_mlp.R](https://github.com/rstudio/tfruns/blob/main/inst/examples/mnist_mlp/mnist_mlp.R)) trains a Keras model to recognize MNIST digits. 55 | 56 | To train a model with **tfruns**, just use the `training_run()` function in place of the `source()` function to execute your R script. For example: 57 | 58 | ```{r} 59 | library(tfruns) 60 | training_run("mnist_mlp.R") 61 | ``` 62 | 63 | When training is completed, a summary of the run will automatically be displayed if you are within an interactive R session: 64 | 65 | 66 | 67 | The metrics and output of each run are automatically captured within a *run directory* which is unique for each run that you initiate. Note that for Keras and TF Estimator models this data is captured automatically (no changes to your source code are required). 68 | 69 | You can call the `latest_run()` function to view the results of the last run (including the path to the run directory which stores all of the run's output): 70 | 71 | ```{r} 72 | latest_run() 73 | ``` 74 | ``` 75 | $ run_dir : chr "runs/2017-10-02T14-23-38Z" 76 | $ eval_loss : num 0.0956 77 | $ eval_acc : num 0.98 78 | $ metric_loss : num 0.0624 79 | $ metric_acc : num 0.984 80 | $ metric_val_loss : num 0.0962 81 | $ metric_val_acc : num 0.98 82 | $ flag_dropout1 : num 0.4 83 | $ flag_dropout2 : num 0.3 84 | $ samples : int 48000 85 | $ validation_samples: int 12000 86 | $ batch_size : int 128 87 | $ epochs : int 20 88 | $ epochs_completed : int 20 89 | $ metrics : chr "(metrics data frame)" 90 | $ model : chr "(model summary)" 91 | $ loss_function : chr "categorical_crossentropy" 92 | $ optimizer : chr "RMSprop" 93 | $ learning_rate : num 0.001 94 | $ script : chr "mnist_mlp.R" 95 | $ start : POSIXct[1:1], format: "2017-10-02 14:23:38" 96 | $ end : POSIXct[1:1], format: "2017-10-02 14:24:24" 97 | $ completed : logi TRUE 98 | $ output : chr "(script ouptut)" 99 | $ source_code : chr "(source archive)" 100 | $ context : chr "local" 101 | $ type : chr "training" 102 | ``` 103 | 104 | The run directory used in the example above is "runs/2017-10-02T14-23-38Z". Run directories are by default generated within the "runs" subdirectory of the current working directory, and use a timestamp as the name of the run directory. You can view the report for any given run using the `view_run()` function: 105 | 106 | ```{r} 107 | view_run("runs/2017-10-02T14-23-38Z") 108 | ``` 109 | 110 | ## Comparing Runs 111 | 112 | Let's make a couple of changes to our training script to see if we can improve model performance. We'll change the number of units in our first dense layer to 128, change the `learning_rate` from 0.001 to 0.003 and run 30 rather than 20 `epochs`. After making these changes to the source code we re-run the script using `training_run()` as before: 113 | 114 | ```{r} 115 | training_run("mnist_mlp.R") 116 | ``` 117 | 118 | This will also show us a report summarizing the results of the run, but what we are really interested in is a comparison between this run and the previous one. We can view a comparison via the `compare_runs()` function: 119 | 120 | ```{r} 121 | compare_runs() 122 | ``` 123 | 124 | 125 | 126 | The comparison report shows the model attributes and metrics side-by-side, as well as differences in the source code and output of the training script. 127 | 128 | Note that `compare_runs()` will by default compare the last two runs, however you can pass any two run directories you like to be compared. 129 | 130 | ## Analyzing Runs 131 | 132 | We've demonstrated visualizing and comparing one or two runs, however as you accumulate more runs you'll generally want to analyze and compare runs many runs. You can use the `ls_runs()` function to yield a data frame with summary information on all of the runs you've conducted within a given directory: 133 | 134 | ```{r} 135 | ls_runs() 136 | ``` 137 | ``` 138 | Data frame: 4 x 28 139 | run_dir eval_loss eval_acc metric_loss metric_acc metric_val_loss metric_val_acc 140 | 1 runs/2017-12-09T21-01-11Z 0.1485 0.9562 0.2577 0.9240 0.1482 0.9545 141 | 2 runs/2017-12-09T21-00-11Z 0.1438 0.9573 0.2655 0.9208 0.1505 0.9559 142 | 3 runs/2017-12-09T19-59-44Z 0.1407 0.9580 0.2597 0.9241 0.1402 0.9578 143 | 4 runs/2017-12-09T19-56-48Z 0.1437 0.9555 0.2610 0.9227 0.1459 0.9551 144 | # ... with 21 more columns: 145 | # flag_batch_size, flag_dropout1, flag_dropout2, samples, validation_samples, batch_size, 146 | # epochs, epochs_completed, metrics, model, loss_function, optimizer, learning_rate, script, 147 | # start, end, completed, output, source_code, context, type 148 | ``` 149 | 150 | You can also render a sortable, filterable version all of the columns within RStudio using the `View()` function: 151 | 152 | ```{r} 153 | View(ls_runs()) 154 | ``` 155 | 156 | 157 | 158 | The `ls_runs()` function also supports `subset` and `order` arguments. For example, the following will yield all runs with an eval accuracy better than 0.98: 159 | 160 | ```{r} 161 | ls_runs(eval_acc > 0.9570, order = eval_acc) 162 | ``` 163 | ``` 164 | Data frame: 2 x 28 165 | run_dir eval_acc eval_loss metric_loss metric_acc metric_val_loss metric_val_acc 166 | 1 runs/2017-12-09T19-59-44Z 0.9580 0.1407 0.2597 0.9241 0.1402 0.9578 167 | 2 runs/2017-12-09T21-00-11Z 0.9573 0.1438 0.2655 0.9208 0.1505 0.9559 168 | # ... with 21 more columns: 169 | # flag_batch_size, flag_dropout1, flag_dropout2, samples, validation_samples, batch_size, 170 | # epochs, epochs_completed, metrics, model, loss_function, optimizer, learning_rate, script, 171 | # start, end, completed, output, source_code, context, type 172 | ``` 173 | 174 | You can pass the results of `ls_runs()` to compare runs (which will always compare the first two runs passed). For example, this will compare the two runs that performed best in terms of evaluation accuracy: 175 | 176 | ```{r} 177 | compare_runs(ls_runs(eval_acc > 0.9570, order = eval_acc)) 178 | ``` 179 | 180 | 181 | 182 | ## RStudio IDE 183 | 184 | If you use RStudio with **tfruns**, it's strongly recommended that you use RStudio v1.1 or higher, as there are are a number of points of integration with the IDE that require this newer releases. 185 | 186 | ### Addin 187 | 188 | The **tfruns** package installs an RStudio IDE addin which provides quick access to frequently used functions from the Addins menu: 189 | 190 | 191 | 192 | Note that you can use **Tools** -> **Modify Keyboard Shortcuts** within RStudio to assign a keyboard shortcut to one or more of the addin commands. 193 | 194 | ### Background Training 195 | 196 | RStudio v1.1 includes a Terminal pane alongside the Console pane. Since training runs can become quite lengthy, it's often useful to run them in the background in order to keep the R console free for other work. You can do this from a Terminal as follows: 197 | 198 | 199 | 200 | If you are not running within RStudio then you can of course use a system terminal window for background training. 201 | 202 | ### Publishing Reports 203 | 204 | Training run views and comparisons are HTML documents which can be saved and shared with others. When viewing a report within RStudio v1.1 you can save a copy of the report or publish it to RPubs or RStudio Connect: 205 | 206 | 207 | 208 | If you are not running within RStudio then you can use the `save_run_view()` and `save_run_comparison()` functions to create standalone HTML versions of run reports. 209 | 210 | 211 | ## Hyperparameter Tuning 212 | 213 | Tuning a model often requires exploring the impact of changes to many hyperparameters. The best way to approach this is generally to systematically train over the combinations of those parameters to determine which combination yields the best model. See the [Hyperparmeter Tuning](https://tensorflow.rstudio.com/tools/tfruns/articles/tuning.html) article for details on how to accomplish this with tfruns. 214 | 215 | ## Managing Runs 216 | 217 | There are a variety of tools available for managing training run output, including: 218 | 219 | 1) Exporting run artifacts (e.g. saved models). 220 | 221 | 2) Copying and purging run directories. 222 | 223 | 3) Using a custom run directory for an experiment or other set of related runs. 224 | 225 | The [Managing Runs](https://tensorflow.rstudio.com/tools/tfruns/articles/managing.html) article provides additional details on using these features. 226 | -------------------------------------------------------------------------------- /vignettes/tuning.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Hyperparameter Tuning" 3 | output: 4 | rmarkdown::html_vignette: default 5 | vignette: > 6 | %\VignetteIndexEntry{Hyperparameter Tuning} 7 | %\VignetteEngine{knitr::rmarkdown} 8 | %\VignetteEncoding{UTF-8} 9 | type: docs 10 | repo: https://github.com/rstudio/tfruns 11 | menu: 12 | main: 13 | name: "Hyperparameter Tuning" 14 | identifier: "tools-tfruns-tuning" 15 | parent: "tfruns-top" 16 | weight: 20 17 | --- 18 | 19 | ```{r setup, include=FALSE} 20 | knitr::opts_chunk$set(eval = FALSE) 21 | ``` 22 | 23 | 24 | ## Overview 25 | 26 | Tuning a model often requires exploring the impact of changes to many hyperparameters. The best way to approach this is generally not by changing the source code of the training script as we did above, but instead by defining flags for key parameters then training over the combinations of those flags to determine which combination of flags yields the best model. 27 | 28 | ## Training Flags 29 | 30 | Here's a declaration of 2 flags that control dropout rate within a model: 31 | 32 | ```{r} 33 | FLAGS <- flags( 34 | flag_numeric("dropout1", 0.4), 35 | flag_numeric("dropout2", 0.3) 36 | ) 37 | ``` 38 | 39 | These flags are then used in the definition of the model here: 40 | 41 | ```{r} 42 | model <- keras_model_sequential() 43 | model %>% 44 | layer_dense(units = 128, activation = 'relu', input_shape = c(784)) %>% 45 | layer_dropout(rate = FLAGS$dropout1) %>% 46 | layer_dense(units = 128, activation = 'relu') %>% 47 | layer_dropout(rate = FLAGS$dropout2) %>% 48 | layer_dense(units = 10, activation = 'softmax') 49 | ``` 50 | 51 | Once we've defined flags, we can pass alternate flag values to `training_run()` as follows: 52 | 53 | ```{r} 54 | training_run('mnist_mlp.R', flags = list(dropout1 = 0.2, dropout2 = 0.2)) 55 | ``` 56 | 57 | You aren't required to specify all of the flags (any flags excluded will simply use their default value). 58 | 59 | Flags make it very straightforward to systematically explore the impact of changes to hyperparameters on model performance, for example: 60 | 61 | ```{r} 62 | for (dropout1 in c(0.1, 0.2, 0.3)) 63 | training_run('mnist_mlp.R', flags = list(dropout1 = dropout1)) 64 | ``` 65 | 66 | Flag values are automatically included in run data with a "flag_" prefix (e.g. `flag_dropout1`, `flag_dropout2`). 67 | 68 | See the article on [training flags](https://tensorflow.rstudio.com/tools/tfruns/overview/) for additional documentation on using flags. 69 | 70 | ## Tuning Runs 71 | 72 | Above we demonstrated writing a loop to call `training_run()` with various different flag values. A better way to accomplish this is the `tuning_run()` function, which allows you to specify multiple values for each flag, and executes training runs for all combinations of the specified flags. For example: 73 | 74 | ```{r} 75 | # run various combinations of dropout1 and dropout2 76 | runs <- tuning_run("mnist_mlp.R", flags = list( 77 | dropout1 = c(0.2, 0.3, 0.4), 78 | dropout2 = c(0.2, 0.3, 0.4) 79 | )) 80 | 81 | # find the best evaluation accuracy 82 | runs[order(runs$eval_acc, decreasing = TRUE), ] 83 | ``` 84 | ``` 85 | Data frame: 9 x 28 86 | run_dir eval_loss eval_acc metric_loss metric_acc metric_val_loss metric_val_acc 87 | 9 runs/2018-01-26T13-21-03Z 0.1002 0.9817 0.0346 0.9900 0.1086 0.9794 88 | 6 runs/2018-01-26T13-23-26Z 0.1133 0.9799 0.0409 0.9880 0.1236 0.9778 89 | 5 runs/2018-01-26T13-24-11Z 0.1056 0.9796 0.0613 0.9826 0.1119 0.9777 90 | 4 runs/2018-01-26T13-24-57Z 0.1098 0.9788 0.0868 0.9770 0.1071 0.9771 91 | 2 runs/2018-01-26T13-26-28Z 0.1185 0.9783 0.0688 0.9819 0.1150 0.9783 92 | 3 runs/2018-01-26T13-25-43Z 0.1238 0.9782 0.0431 0.9883 0.1246 0.9779 93 | 8 runs/2018-01-26T13-21-53Z 0.1064 0.9781 0.0539 0.9843 0.1086 0.9795 94 | 7 runs/2018-01-26T13-22-40Z 0.1043 0.9778 0.0796 0.9772 0.1094 0.9777 95 | 1 runs/2018-01-26T13-27-14Z 0.1330 0.9769 0.0957 0.9744 0.1304 0.9751 96 | # ... with 21 more columns: 97 | # flag_batch_size, flag_dropout1, flag_dropout2, samples, validation_samples, batch_size, 98 | # epochs, epochs_completed, metrics, model, loss_function, optimizer, learning_rate, script, 99 | # start, end, completed, output, source_code, context, type 100 | ``` 101 | 102 | Note that the `tuning_run()` function returns a data frame containing a summary of all of the executed training runs. 103 | 104 | ## Experiment Scopes 105 | 106 | By default all runs go into the "runs" sub-directory of the current working directory. For various types of ad-hoc experimentation this works well, but in some cases for a tuning run you may want to create a separate directory scope. 107 | 108 | You can do this by specifying the `runs_dir` argument: 109 | 110 | ```{r} 111 | # run various combinations of dropout1 and dropout2 112 | tuning_run("mnist_mlp.R", runs_dir = "dropout_tuning", flags = list( 113 | dropout1 = c(0.2, 0.3, 0.4), 114 | dropout2 = c(0.2, 0.3, 0.4) 115 | )) 116 | 117 | # list runs witin the specified runs_dir 118 | ls_runs(order = eval_acc, runs_dir = "dropout_tuning") 119 | ``` 120 | ``` 121 | Data frame: 9 x 28 122 | run_dir eval_acc eval_loss metric_loss metric_acc metric_val_loss metric_val_acc 123 | 9 dropout_tuning/2018-01-26T13-38-02Z 0.9803 0.0980 0.0324 0.9902 0.1096 0.9789 124 | 6 dropout_tuning/2018-01-26T13-40-40Z 0.9795 0.1243 0.0396 0.9885 0.1341 0.9784 125 | 2 dropout_tuning/2018-01-26T13-43-55Z 0.9791 0.1138 0.0725 0.9813 0.1205 0.9773 126 | 7 dropout_tuning/2018-01-26T13-39-49Z 0.9786 0.1027 0.0796 0.9778 0.1053 0.9761 127 | 3 dropout_tuning/2018-01-26T13-43-08Z 0.9784 0.1206 0.0479 0.9871 0.1246 0.9775 128 | 4 dropout_tuning/2018-01-26T13-42-21Z 0.9784 0.1026 0.0869 0.9766 0.1108 0.9769 129 | 5 dropout_tuning/2018-01-26T13-41-31Z 0.9783 0.1086 0.0589 0.9832 0.1216 0.9764 130 | 8 dropout_tuning/2018-01-26T13-38-57Z 0.9780 0.1007 0.0511 0.9855 0.1100 0.9771 131 | 1 dropout_tuning/2018-01-26T13-44-41Z 0.9770 0.1178 0.1017 0.9734 0.1244 0.9757 132 | # ... with 21 more columns: 133 | # flag_batch_size, flag_dropout1, flag_dropout2, samples, validation_samples, batch_size, epochs, 134 | # epochs_completed, metrics, model, loss_function, optimizer, learning_rate, script, start, end, 135 | # completed, output, source_code, context, type 136 | ``` 137 | 138 | ## Sampling Flag Combinations 139 | 140 | If the number of flag combinations is very large, you can also specify that only a random sample of combinations should be tried using the `sample` parameter. For example: 141 | 142 | ```{r} 143 | # run random sample (0.3) of dropout1 and dropout2 combinations 144 | runs <- tuning_run("mnist_mlp.R", sample = 0.3, flags = list( 145 | dropout1 = c(0.2, 0.3, 0.4), 146 | dropout2 = c(0.2, 0.3, 0.4) 147 | )) 148 | ``` 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | --------------------------------------------------------------------------------