├── .Rbuildignore ├── .covrignore ├── .gitignore ├── .lintr ├── .travis.yml ├── DESCRIPTION ├── LICENCE ├── LICENSE.md ├── NAMESPACE ├── NEWS.md ├── R ├── cache-abstract.R ├── cache-file.R ├── cache-functions.R ├── cache-memory-file.R ├── cache-memory.R ├── constants.R ├── eddy-functions.R ├── eddy-r6.R ├── flow-dfg.R ├── flow-dfr.R ├── flow-functions.R ├── flow-r6.R ├── helper.R ├── nmisc.R ├── package.R ├── sink-ns.R └── source-file.R ├── README.md ├── _pkgdown.yml ├── codecov.yml ├── cran-comments.md ├── docs ├── .nojekyll ├── LICENSE-text.html ├── articles │ ├── index.html │ └── rflow-vignette.html ├── authors.html ├── docsearch.css ├── docsearch.js ├── index.html ├── link.svg ├── news │ └── index.html ├── pkgdown.css ├── pkgdown.js ├── pkgdown.yml └── reference │ ├── cache_file.html │ ├── cache_memory.html │ ├── cache_memory_file.html │ ├── collect.R6Flow.html │ ├── compute.R6Flow.html │ ├── default_cache.html │ ├── default_eddy_env.html │ ├── delete_eddy.html │ ├── element.html │ ├── flow_call.html │ ├── flow_dfg.html │ ├── flow_dfr.html │ ├── flow_file_source.html │ ├── flow_fn.html │ ├── flow_ns_sink.html │ ├── flow_options.html │ ├── forget.html │ ├── get_current_eddy.html │ ├── get_eddy.html │ ├── imports.html │ ├── index.html │ ├── is_current.html │ ├── is_flow.html │ ├── is_flow_fn.html │ ├── is_not_flow_fn.html │ ├── is_valid.html │ ├── make_flow_fn.html │ ├── new_eddy.html │ ├── set_current_eddy.html │ ├── to_ns.html │ └── use_eddy.html ├── man ├── cache_file.Rd ├── cache_memory.Rd ├── cache_memory_file.Rd ├── collect.R6Flow.Rd ├── compute.R6Flow.Rd ├── default_cache.Rd ├── default_eddy_env.Rd ├── delete_eddy.Rd ├── element.Rd ├── flow_call.Rd ├── flow_dfg.Rd ├── flow_dfr.Rd ├── flow_file_source.Rd ├── flow_fn.Rd ├── flow_ns_sink.Rd ├── flow_options.Rd ├── forget.Rd ├── get_current_eddy.Rd ├── get_eddy.Rd ├── imports.Rd ├── is_current.Rd ├── is_flow.Rd ├── is_flow_fn.Rd ├── is_not_flow_fn.Rd ├── is_valid.Rd ├── make_flow_fn.Rd ├── new_eddy.Rd ├── set_current_eddy.Rd ├── to_ns.Rd └── use_eddy.Rd ├── rflow.Rproj ├── tests ├── testthat.R └── testthat │ ├── test-cache-file.R │ ├── test-cache-memory-file.R │ ├── test-cache-memory.R │ ├── test-eddy-functions.R │ ├── test-flow-dfg.R │ ├── test-flow-dfr.R │ ├── test-flow-functions.R │ ├── test-helper.R │ ├── test-integrated.R │ ├── test-sink-ns.R │ └── test-source-file.R └── vignettes └── introduction.Rmd /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | ^\.travis\.yml$ 4 | ^codecov\.yml$ 5 | ^\.covrignore$ 6 | ^\.lintr$ 7 | ^_pkgdown\.yml$ 8 | ^docs$ 9 | ^codemeta\.json$ 10 | ^cran-comments\.md$ 11 | ^LICENSE\.md$ 12 | -------------------------------------------------------------------------------- /.covrignore: -------------------------------------------------------------------------------- 1 | R/nmisc.R 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # RStudio 2 | .Rproj.user 3 | .Rhistory 4 | .RData 5 | .Ruserdata 6 | 7 | # History files 8 | .Rapp.history 9 | # Session Data files 10 | # Example code in package build process 11 | *-Ex.R 12 | # Output files from R CMD build 13 | /*.tar.gz 14 | # Output files from R CMD check 15 | /*.Rcheck/ 16 | # RStudio files 17 | .Rproj.user/ 18 | # produced vignettes 19 | vignettes/*.html 20 | vignettes/*.pdf 21 | inst/doc 22 | # OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 23 | .httr-oauth 24 | # knitr and R markdown default cache directories 25 | /*_cache/ 26 | /cache/ 27 | # Temporary files created by R markdown 28 | *.utf8.md 29 | *.knit.md 30 | 31 | # macOS files 32 | .DS_Store 33 | *.swp 34 | 35 | # project cache directory 36 | cache/ 37 | -------------------------------------------------------------------------------- /.lintr: -------------------------------------------------------------------------------- 1 | linters: with_defaults( 2 | object_usage_linter = NULL 3 | ) 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # R for travis: see documentation at https://docs.travis-ci.com/user/languages/r 2 | 3 | language: R 4 | sudo: false 5 | cache: packages 6 | 7 | r: 8 | - 3.4 9 | - release 10 | - devel 11 | 12 | # coverage for the release version 13 | after_success: 14 | - test $TRAVIS_R_VERSION_STRING = "release" && Rscript -e 'covr::codecov()' 15 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: rflow 2 | Type: Package 3 | Title: Flexible R Pipelines with Caching 4 | Version: 0.5.3 5 | Authors@R: c( 6 | person("Mike", "Badescu", email = "mike.badescu@numeract.com", role = c("aut", "cre")), 7 | person("Ana-Maria", "Niculescu", role = "aut"), 8 | person("Teodor", "Ciuraru", role = "ctb"), 9 | person("Numeract LLC", role = "cph") 10 | ) 11 | Description: Cache function output to prevent re-calculation when called again 12 | with the same arguments. Transformed (rflow) functions can be chained to 13 | create pipelines by avoiding excessive re-hashing. 14 | URL: https://github.com/numeract/rflow 15 | BugReports: https://github.com/numeract/rflow/issues 16 | License: MIT + file LICENCE 17 | Encoding: UTF-8 18 | LazyData: true 19 | RoxygenNote: 6.0.1.9000 20 | Depends: 21 | R (>= 3.4) 22 | Imports: 23 | crayon, 24 | digest, 25 | dplyr, 26 | forcats, 27 | fs, 28 | magrittr, 29 | purrr, 30 | R6, 31 | rlang, 32 | tibble 33 | Suggests: 34 | testthat, 35 | covr, 36 | shiny, 37 | knitr, 38 | rmarkdown 39 | VignetteBuilder: knitr 40 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | YEAR: 2018 2 | COPYRIGHT HOLDER: Numeract LLC 3 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | Copyright (c) 2018 Numeract LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | S3method("[",R6Flow) 4 | S3method(collect,Element) 5 | S3method(collect,R6Flow) 6 | S3method(compute,Element) 7 | S3method(compute,R6Flow) 8 | export("%>%") 9 | export("%||%") 10 | export(cache_file) 11 | export(cache_memory) 12 | export(cache_memory_file) 13 | export(collect) 14 | export(compute) 15 | export(default_cache) 16 | export(default_eddy_env) 17 | export(default_flow_options) 18 | export(delete_eddy) 19 | export(element) 20 | export(flow_call) 21 | export(flow_dfg) 22 | export(flow_dfr) 23 | export(flow_file_source) 24 | export(flow_fn) 25 | export(flow_ns_sink) 26 | export(forget) 27 | export(get_current_eddy) 28 | export(get_eddy) 29 | export(get_flow_options) 30 | export(is_current) 31 | export(is_flow) 32 | export(is_flow_fn) 33 | export(is_valid) 34 | export(make_flow_fn) 35 | export(new_eddy) 36 | export(set_current_eddy) 37 | export(set_flow_options) 38 | export(use_eddy) 39 | importFrom(dplyr,collect) 40 | importFrom(dplyr,compute) 41 | importFrom(magrittr,"%>%") 42 | importFrom(purrr,"%||%") 43 | -------------------------------------------------------------------------------- /NEWS.md: -------------------------------------------------------------------------------- 1 | # rflow 0.5.3 2 | 3 | - added `compute()` function 4 | - misc fixes (mostly `flow_dfr` and `flow_dfg`) and more tests 5 | - fix `flow_dfr` and `flow_dfg` for factors 6 | 7 | 8 | # rflow 0.5.0 9 | 10 | - the function supplied to `flow_dfr` receives only changed rows and no index 11 | - new function: `flow_dfg` hadles grouped data frames 12 | - refactor fn_key formation 13 | - misc fixes 14 | 15 | 16 | # rflow 0.4.1 17 | 18 | - **extensive rewrite, breaks compatibility with previous versions** 19 | - add: separate cache from eddy (new R6 class tree) 20 | - updated eddy functions 21 | - R6 class tree for flow functions 22 | - `flow_` functions for file source and namespace sink 23 | - `flow_dfr` for caching row-wise data frame operations 24 | - alternative setup that does not require `make_flow_fn` functions 25 | - lazy computation (does not re-compute a pipeline if not needed) 26 | - extensive testing 27 | 28 | 29 | # rflow 0.3.5 30 | 31 | - add: is_cached to test whether a state already exists in cache 32 | - add: delete_cache to delete the given state from rflow and eddy 33 | 34 | 35 | # rflow 0.3.4 36 | 37 | - fix: fail to init rflow when no data on disk 38 | - fix: fail when retrieving an rflow element inside cached function 39 | - add: `element()` function for R6Flow objects 40 | 41 | 42 | # rflow 0.3.3 43 | 44 | - `[` method for R6Flow to extract its elements 45 | 46 | 47 | # rflow 0.3.2 48 | 49 | - fix: make_ns_sink changed to follow make_sink pattern 50 | 51 | 52 | # rflow 0.3.1 53 | 54 | - fix: eddies always create cache_env and dir on disk 55 | - fix: eddies require registered rflow when adding data 56 | 57 | 58 | # rflow 0.3.0 59 | 60 | - added file source 61 | - add multi-purpose sink 62 | 63 | 64 | # rflow 0.2.1 65 | 66 | - collect is now an S3 method, using the S3 generic from dplyr 67 | 68 | 69 | # rflow 0.2.0 70 | 71 | - new functions: new_eddy, get_eddy, delete_eddy 72 | - added tests 73 | - added R6 obj printing 74 | - update documentation 75 | - update memory and disk caching issues 76 | 77 | 78 | # rflow 0.1.0 79 | 80 | First release 81 | -------------------------------------------------------------------------------- /R/cache-abstract.R: -------------------------------------------------------------------------------- 1 | # cache engine (abstract methods - eddy calls only these) 2 | 3 | 4 | # !diagnostics suppress=., self, private 5 | 6 | 7 | # nocov start 8 | R6Cache <- R6::R6Class( 9 | classname = "R6Cache", 10 | public = list( 11 | initialize = function() {stop("abstract method")}, 12 | 13 | list_groups = function() {stop("abstract method")}, 14 | has_group = function(group) {stop("abstract method")}, 15 | # no get_group() function 16 | add_group = function(group) {stop("abstract method")}, 17 | forget_group = function(group) {stop("abstract method")}, 18 | delete_group = function(group) {stop("abstract method")}, 19 | 20 | list_keys = function(group) {stop("abstract method")}, 21 | has_key = function(group, key) {stop("abstract method")}, 22 | 23 | get_data = function(group, key) {stop("abstract method")}, 24 | add_data = function(group, key, value) {stop("abstract method")}, 25 | delete_data = function(group, key) {stop("abstract method")}, 26 | 27 | summary = function() {stop("abstract method")}, 28 | print = function() {}, 29 | reset = function() {stop("abstract method")}, 30 | terminate = function() {stop("abstract method")} 31 | ) 32 | ) 33 | 34 | 35 | # print ---- 36 | R6Cache$set("public", "print", function() { 37 | 38 | df <- self$summary() 39 | 40 | emph_obj <- paste0("<", crayon::italic(class(self)[[1L]]), ">") 41 | cat(emph_obj, "with", crayon::bold(nrow(df)), "fn_keys\n") 42 | print(df) 43 | 44 | invisible(self) 45 | }, overwrite = TRUE) 46 | # nocov end 47 | -------------------------------------------------------------------------------- /R/cache-file.R: -------------------------------------------------------------------------------- 1 | # cache engine for file only 2 | 3 | 4 | # !diagnostics suppress=., self, private 5 | 6 | 7 | R6CacheFile <- R6::R6Class( 8 | classname = "R6CacheFile", 9 | inherit = R6Cache, 10 | public = list( 11 | cache_dir = NULL 12 | ) 13 | ) 14 | 15 | 16 | # initialize ---- 17 | R6CacheFile$set("public", "initialize", function(cache_dir) { 18 | 19 | stopifnot(rlang::is_string(cache_dir)) 20 | 21 | # store the absolute, not relative path for cache 22 | cache_dir <- fs::path_abs(cache_dir) 23 | if (!fs::dir_exists(cache_dir)) { 24 | fs::dir_create(cache_dir) 25 | } 26 | self$cache_dir <- cache_dir 27 | 28 | invisible(NULL) 29 | }, overwrite = TRUE) 30 | 31 | 32 | # list_groups ---- 33 | R6CacheFile$set("public", "list_groups", function() { 34 | 35 | # error if cache_dir does not exist 36 | as.character( 37 | fs::path_file(fs::dir_ls(self$cache_dir, type = "directory"))) 38 | 39 | }, overwrite = TRUE) 40 | 41 | 42 | # has_group ---- 43 | R6CacheFile$set("public", "has_group", function(group) { 44 | 45 | require_keys(group) 46 | 47 | group_dir <- fs::path(self$cache_dir, group) 48 | fs::dir_exists(group_dir) 49 | }, overwrite = TRUE) 50 | 51 | 52 | # add_group ---- 53 | R6CacheFile$set("public", "add_group", function(group) { 54 | 55 | require_keys(group) 56 | stopifnot(fs::dir_exists(self$cache_dir)) 57 | 58 | group_dir <- fs::path(self$cache_dir, group) 59 | if (!fs::dir_exists(group_dir)) { 60 | fs::dir_create(group_dir) 61 | } 62 | 63 | self$has_group(group) 64 | }, overwrite = TRUE) 65 | 66 | 67 | # forget_group ---- 68 | R6CacheFile$set("public", "forget_group", function(group) { 69 | 70 | require_keys(group) 71 | stopifnot(fs::dir_exists(self$cache_dir)) 72 | 73 | # this also adds the group on disk, if missing 74 | group_dir <- fs::path(self$cache_dir, group) 75 | if (fs::dir_exists(group_dir)) { 76 | unlink(group_dir, recursive = TRUE, force = DIR_DELETE_FORCE) 77 | Sys.sleep(DIR_DELETE_WAIT) 78 | } 79 | fs::dir_create(group_dir) 80 | 81 | self$has_group(group) && length(fs::dir_ls(group_dir)) == 0L 82 | }, overwrite = TRUE) 83 | 84 | 85 | # delete_group ---- 86 | R6CacheFile$set("public", "delete_group", function(group) { 87 | 88 | require_keys(group) 89 | stopifnot(fs::dir_exists(self$cache_dir)) 90 | 91 | group_dir <- fs::path(self$cache_dir, group) 92 | if (fs::dir_exists(group_dir)) { 93 | unlink(group_dir, recursive = TRUE, force = DIR_DELETE_FORCE) 94 | # Sys.sleep(DIR_DELETE_WAIT) 95 | } 96 | 97 | !self$has_group(group) 98 | }, overwrite = TRUE) 99 | 100 | 101 | # list_keys ---- 102 | R6CacheFile$set("public", "list_keys", function(group) { 103 | 104 | require_keys(group) 105 | stopifnot(fs::dir_exists(self$cache_dir)) 106 | 107 | # no error if group NOT present on disk 108 | group_dir <- fs::path(self$cache_dir, group) 109 | if (fs::dir_exists(group_dir)) { 110 | as.character(fs::path_file(fs::dir_ls(group_dir, type = "file"))) 111 | } else { 112 | character(0L) 113 | } 114 | }, overwrite = TRUE) 115 | 116 | 117 | # has_key ---- 118 | R6CacheFile$set("public", "has_key", function(group, key) { 119 | 120 | require_keys(group, key) 121 | 122 | key %in% self$list_keys(group) 123 | }, overwrite = TRUE) 124 | 125 | 126 | # get_data ---- 127 | R6CacheFile$set("public", "get_data", function(group, key) { 128 | 129 | require_keys(group, key) 130 | stopifnot(fs::dir_exists(self$cache_dir)) 131 | # add group only if not already present 132 | self$add_group(group) 133 | 134 | key_path <- fs::path(self$cache_dir, group, key) 135 | stopifnot(fs::file_exists(key_path)) 136 | 137 | readRDS(key_path) 138 | }, overwrite = TRUE) 139 | 140 | 141 | # add_data ---- 142 | R6CacheFile$set("public", "add_data", function(group, key, value) { 143 | 144 | require_keys(group, key) 145 | # add group only if not already present 146 | self$add_group(group) 147 | 148 | key_path <- fs::path(self$cache_dir, group, key) 149 | saveRDS(value, key_path) 150 | 151 | self$has_key(group, key) 152 | }, overwrite = TRUE) 153 | 154 | 155 | # delete_data ---- 156 | R6CacheFile$set("public", "delete_data", function(group, key) { 157 | 158 | require_keys(group, key) 159 | # add group only if not already present 160 | self$add_group(group) 161 | 162 | key_path <- fs::path(self$cache_dir, group, key) 163 | if (fs::file_exists(key_path)) { 164 | fs::file_delete(key_path) 165 | } 166 | 167 | !self$has_key(group, key) 168 | }, overwrite = TRUE) 169 | 170 | 171 | # summary ---- 172 | R6CacheFile$set("public", "summary", function() { 173 | 174 | groups <- self$list_groups() 175 | n_keys <- groups %>% 176 | purrr::map_int(~ length(self$list_keys(.))) 177 | df <- tibble::tibble( 178 | fn_key = groups, 179 | on_disk = n_keys 180 | ) 181 | 182 | df 183 | }, overwrite = TRUE) 184 | 185 | 186 | # reset ---- 187 | R6CacheFile$set("public", "reset", function() { 188 | # the instance is as if just initialized 189 | 190 | if (fs::dir_exists(self$cache_dir)) { 191 | unlink(self$cache_dir, recursive = TRUE, force = DIR_DELETE_FORCE) 192 | Sys.sleep(DIR_DELETE_WAIT) 193 | } 194 | fs::dir_create(self$cache_dir) 195 | 196 | invisible(self) 197 | }, overwrite = TRUE) 198 | 199 | 200 | # terminate ---- 201 | R6CacheFile$set("public", "terminate", function() { 202 | # reset + delete its own data structures, e.g. folders 203 | # object cannot be used afterwards 204 | 205 | if (fs::dir_exists(self$cache_dir)) { 206 | unlink(self$cache_dir, recursive = TRUE, force = DIR_DELETE_FORCE) 207 | } 208 | self$cache_dir <- NULL 209 | 210 | invisible(NULL) 211 | }, overwrite = TRUE) 212 | -------------------------------------------------------------------------------- /R/cache-functions.R: -------------------------------------------------------------------------------- 1 | # cache functions, some accessible to the user 2 | 3 | 4 | # !diagnostics suppress=., 5 | 6 | 7 | #' Get the memory only cache engine. 8 | #' 9 | #' All cache data will be stored only in memory. 10 | #' 11 | #' @return A cache object that inherits from \code{R6Cache}. 12 | #' 13 | #' @family cache functions 14 | #' 15 | #' @examples 16 | #' cache_in_memory <- cache_memory() 17 | #' 18 | #' @export 19 | cache_memory <- function() { 20 | 21 | R6CacheMemory$new() 22 | } 23 | 24 | 25 | #' Get the file only cache engine. 26 | #' 27 | #' All cache data will be stored only on the local disk. 28 | #' 29 | #' @param cache_dir Directory where to store the cache files. 30 | #' 31 | #' @return A cache object that inherits from \code{R6Cache}. 32 | #' 33 | #' @family cache functions 34 | #' 35 | #' @examples 36 | #' \dontrun{ 37 | #' cache_in_file <- cache_file("path_to_cache_diretory") 38 | #' } 39 | #' 40 | #' @export 41 | cache_file <- function(cache_dir) { 42 | 43 | R6CacheFile$new(cache_dir) 44 | } 45 | 46 | 47 | #' Get the memory-file cache engine. 48 | #' 49 | #' The cache data will be stored both in memory (for quick access) and on 50 | #' disk (for persistence). 51 | #' 52 | #' @param cache_dir Directory where to store the cache files. 53 | #' 54 | #' @return A cache object that inherits from \code{R6Cache}. 55 | #' 56 | #' @family cache functions 57 | #' 58 | #' @examples 59 | #' \dontrun{ 60 | #' cache_fmem <- cache_memory_file("path_to_cache_diretory") 61 | #' } 62 | #' 63 | #' 64 | #' @export 65 | cache_memory_file <- function(cache_dir) { 66 | 67 | R6CacheMemoryFile$new(cache_dir) 68 | } 69 | 70 | 71 | #' Get the default cache engine. 72 | #' 73 | #' Currently, the default cache engine is \code{cache_memory()}, but this 74 | #' might change in the future. 75 | #' 76 | #' @return A cache object that inherits from \code{R6Cache}. 77 | #' 78 | #' @family cache functions 79 | #' 80 | #' @examples 81 | #' \dontrun{ 82 | #' current_cache <- default_cache() 83 | #' } 84 | #' 85 | #' @export 86 | default_cache <- function() { 87 | 88 | cache_memory() 89 | } 90 | -------------------------------------------------------------------------------- /R/cache-memory.R: -------------------------------------------------------------------------------- 1 | # cache engine for memory only 2 | 3 | 4 | # !diagnostics suppress=., self, private 5 | 6 | 7 | R6CacheMemory <- R6::R6Class( 8 | classname = "R6CacheMemory", 9 | inherit = R6Cache, 10 | public = list( 11 | cache_env = NULL 12 | ) 13 | ) 14 | 15 | 16 | # initialize ---- 17 | R6CacheMemory$set("public", "initialize", function() { 18 | 19 | self$cache_env <- new.env(hash = TRUE, parent = emptyenv()) 20 | 21 | invisible(NULL) 22 | }, overwrite = TRUE) 23 | 24 | 25 | # list_groups ---- 26 | R6CacheMemory$set("public", "list_groups", function() { 27 | 28 | # error if cache_env not an environment 29 | as.character(ls.str(pos = self$cache_env, all.names = TRUE)) 30 | }, overwrite = TRUE) 31 | 32 | 33 | # has_group ---- 34 | R6CacheMemory$set("public", "has_group", function(group) { 35 | 36 | require_keys(group) 37 | 38 | base::exists(group, where = self$cache_env, inherits = FALSE) 39 | }, overwrite = TRUE) 40 | 41 | 42 | # add_group ---- 43 | R6CacheMemory$set("public", "add_group", function(group) { 44 | 45 | require_keys(group) 46 | 47 | if (!base::exists(group, where = self$cache_env, inherits = FALSE)) { 48 | base::assign(group, value = list(), pos = self$cache_env) 49 | } 50 | 51 | self$has_group(group) 52 | }, overwrite = TRUE) 53 | 54 | 55 | # forget_group ---- 56 | R6CacheMemory$set("public", "forget_group", function(group) { 57 | 58 | require_keys(group) 59 | 60 | # this also adds the group in memory, if missing 61 | base::assign(group, value = list(), pos = self$cache_env) 62 | 63 | self$has_group(group) && length(self$cache_env[[group]]) == 0L 64 | }, overwrite = TRUE) 65 | 66 | 67 | # delete_group ---- 68 | R6CacheMemory$set("public", "delete_group", function(group) { 69 | 70 | require_keys(group) 71 | 72 | if (base::exists(group, where = self$cache_env, inherits = FALSE)) { 73 | base::rm(list = group, pos = self$cache_env) 74 | } 75 | 76 | !self$has_group(group) 77 | }, overwrite = TRUE) 78 | 79 | 80 | # list_keys ---- 81 | R6CacheMemory$set("public", "list_keys", function(group) { 82 | 83 | require_keys(group) 84 | 85 | # no error if group NOT present in memory 86 | kv_lst <- base::get0( 87 | group, envir = self$cache_env, inherits = FALSE, ifnotfound = list()) 88 | 89 | as.character(names(kv_lst)) 90 | }, overwrite = TRUE) 91 | 92 | 93 | # has_key ---- 94 | R6CacheMemory$set("public", "has_key", function(group, key) { 95 | 96 | require_keys(group, key) 97 | 98 | key %in% self$list_keys(group) 99 | }, overwrite = TRUE) 100 | 101 | 102 | # get_data ---- 103 | R6CacheMemory$set("public", "get_data", function(group, key) { 104 | 105 | require_keys(group, key) 106 | # add group only if not already present 107 | self$add_group(group) 108 | 109 | # error if group not present in memory 110 | kv_lst <- base::get(group, envir = self$cache_env, inherits = FALSE) 111 | 112 | if (!(key %in% names(kv_lst))) { 113 | stop("key ", key, "not found for group ", group) 114 | } 115 | 116 | kv_lst[[key]] 117 | }, overwrite = TRUE) 118 | 119 | 120 | # add_data ---- 121 | R6CacheMemory$set("public", "add_data", function(group, key, value) { 122 | 123 | require_keys(group, key) 124 | # add group only if not already present 125 | self$add_group(group) 126 | 127 | kv_lst <- base::get(group, envir = self$cache_env, inherits = FALSE) 128 | if (is.null(value)) { 129 | kv_lst[key] <- list(NULL) 130 | } else { 131 | kv_lst[[key]] <- value 132 | } 133 | base::assign(group, value = kv_lst, pos = self$cache_env) 134 | 135 | self$has_key(group, key) 136 | }, overwrite = TRUE) 137 | 138 | 139 | # delete_data ---- 140 | R6CacheMemory$set("public", "delete_data", function(group, key) { 141 | 142 | require_keys(group, key) 143 | # add group only if not already present 144 | self$add_group(group) 145 | 146 | kv_lst <- base::get(group, envir = self$cache_env, inherits = FALSE) 147 | kv_lst[[key]] <- NULL 148 | base::assign(group, value = kv_lst, pos = self$cache_env) 149 | 150 | !self$has_key(group, key) 151 | }, overwrite = TRUE) 152 | 153 | 154 | # summary ---- 155 | R6CacheMemory$set("public", "summary", function() { 156 | 157 | groups <- self$list_groups() 158 | n_keys <- groups %>% 159 | purrr::map_int(~ length(self$list_keys(.))) 160 | df <- tibble::tibble( 161 | fn_key = groups, 162 | in_mem = n_keys 163 | ) 164 | 165 | df 166 | }, overwrite = TRUE) 167 | 168 | 169 | # reset ---- 170 | R6CacheMemory$set("public", "reset", function() { 171 | # the instance is as if just initialized 172 | 173 | self$cache_env <- new.env(hash = TRUE, parent = emptyenv()) 174 | 175 | # old $cache_env is now unbound, force gc() to free up memory 176 | gc() 177 | invisible(self) 178 | }, overwrite = TRUE) 179 | 180 | 181 | # terminate ---- 182 | R6CacheMemory$set("public", "terminate", function() { 183 | # reset + delete its own data structures, e.g. folders 184 | # object cannot be used afterwards 185 | 186 | self$cache_env <- NULL 187 | 188 | gc() 189 | invisible(NULL) 190 | }, overwrite = TRUE) 191 | -------------------------------------------------------------------------------- /R/constants.R: -------------------------------------------------------------------------------- 1 | # constants / globals for all functions 2 | 3 | 4 | # force dir delete? 5 | DIR_DELETE_FORCE = TRUE 6 | # how long to wait (in seconds) after deleting a dir and recreating it 7 | DIR_DELETE_WAIT = 0.1 8 | 9 | 10 | # create a separate binding environment to keep eddies 11 | .EDDY_ENV <- new.env(parent = emptyenv()) 12 | # Each eddy_env has a string variable .CURRENT_NAME that stores the name 13 | # of the current eddy. This to avoid duplicate binding of the eddy_end 14 | # to help with gc() 15 | .EDDY_ENV[[".CURRENT_NAME"]] <- NA_character_ 16 | 17 | 18 | # flow 19 | .STATE_KEY = "_state" 20 | 21 | 22 | # flow_dfr 23 | .ROW_CACHE = "_row_cache" 24 | ROW_HASH <- "..row_hash.." 25 | 26 | 27 | # flow_dfg 28 | .ROW_ID <- "..row_id.." 29 | GROUP_HASH <- "..group_hash.." 30 | -------------------------------------------------------------------------------- /R/helper.R: -------------------------------------------------------------------------------- 1 | # helper functions not present in nmisc.R 2 | 3 | 4 | # !diagnostics suppress=., 5 | 6 | 7 | is_key <- function(x) { 8 | 9 | if (typeof(x) != "character") return(FALSE) 10 | if (length(x) != 1L) return(FALSE) 11 | if (x %in% c(NA_character_, "")) return(FALSE) 12 | 13 | TRUE 14 | } 15 | 16 | 17 | require_keys <- function(...) { 18 | 19 | key_lgl <- purrr::map_lgl(list(...), ~ is_key(.)) 20 | 21 | if (sum(!key_lgl) > 0L) { 22 | rlang::abort("key arguments must be valid strings") 23 | } 24 | } 25 | 26 | 27 | make_key <- function(fn_name, fn, fn_id, flow_options, class_name) { 28 | 29 | # fn_name 30 | stopifnot(!is.null(fn_name)) 31 | if (!is.character(fn_name)) { 32 | if (is.symbol(fn_name)) { 33 | fn_name <- as.character(fn_name) 34 | } else if (is.language(fn_name)) { 35 | rlang::abort("Anonymous functions not supported.") 36 | } else { 37 | rlang::abort("Unrecognized `fn_name` data type.") 38 | } 39 | } 40 | require_keys(fn_name, class_name) 41 | # fn 42 | stopifnot(is_not_flow_fn(fn)) 43 | # fn_id 44 | if (is.null(fn_id)) { 45 | is_default_id <- TRUE 46 | } else { 47 | is_default_id <- FALSE 48 | stopifnot(is_key(fn_id) || ( 49 | rlang::is_scalar_integerish(fn_id) 50 | && is.finite(fn_id) && (fn_id >= 1)) 51 | ) 52 | if (rlang::is_scalar_integerish(fn_id)) fn_id <- as.integer(fn_id) 53 | } 54 | # when generated by get_flow_options(), flow_options are valid 55 | if (!is.null(flow_options$eval_arg_fn)) { 56 | fn_formals <- formals(args(fn)) 57 | eval_arg_fn_formals <- formals(args(flow_options$eval_arg_fn)) 58 | stopifnot(identical(fn_formals, eval_arg_fn_formals)) 59 | } 60 | # class_name 61 | require_keys(class_name) 62 | 63 | # useful misc 64 | eddy <- flow_options$eddy 65 | stopifnot(inherits(eddy, "R6Eddy")) 66 | # fn_name can be anything & exists only in R6Flow obj (not saved in cache) 67 | fn_names <- purrr::map_chr(eddy$flow_lst, "fn_name") 68 | 69 | # flow_hash: all but fn_name (always ignored) & fn_id; calc only once 70 | fn_formals <- formals(args(fn)) 71 | arg_chr <- paste( 72 | paste(names(fn_formals), as.character(fn_formals), sep = "="), 73 | collapse = ", ") 74 | body_chr <- as.character(body(fn)) 75 | if (length(body_chr) == 0L) { 76 | body_chr <- format(fn) 77 | } 78 | if (any(grepl("\\.Primitive", body_chr))) { 79 | rlang::abort("Primitive functions not supported.") 80 | } 81 | fo_chr <- format(discard_at(flow_options, "eddy")) 82 | flow_hash <- eddy$digest(c(arg_chr, body_chr, fo_chr, class_name)) 83 | 84 | # if fn_id given ==> simple case, otherwise do some guessing 85 | if (is_default_id) { 86 | # get a list of values to try for fn_id (taking hint from fn_name) 87 | fn_keys <- names(fn_names %if_in% fn_name) 88 | fn_id_lst <- 89 | eddy$flow_lst[fn_keys] %>% 90 | purrr::map("fn_id") 91 | fn_key_lst <- 92 | fn_id_lst %>% 93 | purrr::map(~ eddy$digest(c(flow_hash, .))) %>% 94 | purrr::imap_lgl(~ .x == .y) %>% 95 | purrr::keep( ~ .) %>% 96 | names() 97 | 98 | if (length(fn_key_lst) > 1L) { 99 | rlang::abort(paste0( 100 | "Found multiple flows; please supply `fn_id`.")) 101 | } else if (length(fn_key_lst) == 1L) { 102 | fn_key <- fn_key_lst[[1L]] 103 | fn_id <- fn_id_lst[[fn_key]] 104 | action = "get" 105 | rlang::inform(paste0( 106 | "Reusing cache: fn=", fn_name, 107 | " / fn_id=", fn_id, " / fn_key=", fn_key)) 108 | } else { 109 | # try fn_id = 1L & check all keys; ignore fn_name 110 | fn_keys <- names(fn_names) 111 | fn_id <- 1L 112 | fn_key <- eddy$digest(c(flow_hash, fn_id)) 113 | if (fn_key %in% fn_keys) { 114 | action = "get" 115 | rlang::inform(paste0( 116 | "Reusing cache: fn=", fn_name, 117 | " / fn_id=", fn_id, " / fn_key=", fn_key)) 118 | } else { 119 | action = "new" 120 | rlang::inform(paste0( 121 | "New cache: fn=", fn_name, 122 | " / fn_id=", fn_id, " / fn_key=", fn_key)) 123 | } 124 | } 125 | } else { 126 | # fn_id is given: we do not care about fn_name, do not print messages 127 | fn_keys <- names(fn_names) 128 | fn_key <- eddy$digest(c(flow_hash, fn_id)) 129 | if (fn_key %in% fn_keys) { 130 | action = "get" 131 | } else { 132 | action = "new" 133 | } 134 | } 135 | 136 | list( 137 | action = action, # string: 'new' || 'get' 138 | fn_key = fn_key, # key 139 | fn_name = fn_name, # key 140 | fn_id = fn_id # string/key or positive integer 141 | ) 142 | } 143 | 144 | 145 | parse_call <- function(pos = 2L) { 146 | 147 | # the full call into the caller of this function 148 | parent_call <- match.call( 149 | definition = sys.function(-1L), 150 | call = sys.call(-1L), 151 | expand.dots = TRUE, 152 | envir = parent.frame(3L) 153 | ) 154 | 155 | token <- parent_call[[pos]] 156 | if (is.symbol(token)) { 157 | if (as.character(token) == "." && pos == 2L) { 158 | # assume . from %>% ==> hack 159 | rlang::abort("Pipelines %>% not yet supported.") 160 | unmatched_fn_call <- parent_call[[3L]] 161 | parent <- parent.frame() 162 | unmatched_fn_call[[2L]] <- parent[["fn_call"]] 163 | # TODO: %>% still does not work 164 | # parent[["fn_id"]] <- parent[["flow_options"]] 165 | # parent[["flow_options"]] <- get_flow_options() 166 | } else { 167 | rlang::abort("The first argument must be a function call.") 168 | } 169 | } else if (is.language(token)) { 170 | # un-matched argument of the parent call 171 | # this is the as.is call to function (and its arguments) to be flow-ed 172 | unmatched_fn_call <- token 173 | } else { 174 | rlang::abort("Unrecognized argument type, expected a function call.") 175 | } 176 | 177 | fn <- eval(unmatched_fn_call[[1L]]) 178 | if (!is.function(fn)) { 179 | format_fn_call <- paste(format(unmatched_fn_call), collapse = " ") 180 | rlang::abort(paste("Not a function call:", format_fn_call)) 181 | } 182 | if (any(grepl("\\.Primitive", format(fn)))) { 183 | rlang::abort("Primitive functions not supported.") 184 | } 185 | 186 | # match.call for the function (and its arguments) to be flow-ed 187 | fn_call <- match.call( 188 | definition = fn, 189 | call = unmatched_fn_call, 190 | expand.dots = TRUE, 191 | envir = parent.frame(3L) 192 | ) 193 | 194 | fn_call 195 | } 196 | -------------------------------------------------------------------------------- /R/nmisc.R: -------------------------------------------------------------------------------- 1 | # functions from Nmisc 2 | 3 | seq_nrow <- function(x) seq_len(nrow(x)) 4 | seq_ncol <- function(x) seq_len(ncol(x)) 5 | 6 | 7 | keep_if_in <- function(x, y) { 8 | 9 | x[x %in% y] 10 | } 11 | `%if_in%` <- keep_if_in 12 | 13 | 14 | keep_if_not_in <- function(x, y) { 15 | 16 | x[!(x %in% y)] 17 | } 18 | `%if_not_in%` <- keep_if_not_in 19 | 20 | 21 | keep_at <- function(.x, .at) { 22 | 23 | if (length(.at) == 0L) return(.x[0L]) 24 | if (any(is.na(.at))) stop("`.at` must not contain NA's") 25 | 26 | .p <- if (is.character(.at)) { 27 | names(.x) %in% .at 28 | } else if (is.numeric(.at)) { 29 | seq_along(.x) %in% as.integer(.at) 30 | } else { 31 | stop("`.at` must be character (names) or a numeric (positions)") 32 | } 33 | 34 | purrr::keep(.x, .p) 35 | } 36 | 37 | 38 | discard_at <- function(.x, .at) { 39 | 40 | if (length(.at) == 0L) return(.x) 41 | if (any(is.na(.at))) stop("`.at` must not contain NA's") 42 | 43 | .p <- if (is.character(.at)) { 44 | names(.x) %in% .at 45 | } else if (is.numeric(.at)) { 46 | seq_along(.x) %in% as.integer(.at) 47 | } else { 48 | stop("`.at` must be character (names) or a numeric (positions)") 49 | } 50 | 51 | purrr::discard(.x, .p) 52 | } 53 | 54 | 55 | now_utc <- function(length = 1L) { 56 | 57 | len <- as.integer(length[1L]) 58 | stopifnot(base::length(len) == 1L || len >= 0L) 59 | 60 | if (len == 0L) { 61 | as.POSIXct(character(), tz = "UTC") 62 | } else { 63 | now <- Sys.time() 64 | attr(now, "tzone") <- "UTC" 65 | rep(now, len) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /R/package.R: -------------------------------------------------------------------------------- 1 | #' Imports from other packages 2 | #' 3 | #' @name %>% 4 | #' @rdname imports 5 | #' @keywords internal 6 | #' @export 7 | #' @importFrom magrittr %>% 8 | NULL 9 | 10 | 11 | #' @name %||% 12 | #' @rdname imports 13 | #' @keywords internal 14 | #' @export 15 | #' @importFrom purrr %||% 16 | NULL 17 | -------------------------------------------------------------------------------- /R/source-file.R: -------------------------------------------------------------------------------- 1 | # functions to create file data sources (other than R variables) 2 | 3 | 4 | # !diagnostics suppress=., self, private, super 5 | 6 | 7 | # R6FileSource ---- 8 | R6FileSource <- R6::R6Class( 9 | classname = "R6FileSource", 10 | inherit = R6Flow 11 | ) 12 | 13 | 14 | # calc_in_hash ---- 15 | R6FileSource$set("public", "calc_in_hash_file_source", function( 16 | rf_env = parent.frame() 17 | ) { 18 | file_path <- rf_env$elem_args[[1L]] 19 | in_hashes <- self$eddy$digest_each(objects = file_path, is_file_path = TRUE) 20 | in_hash <- self$eddy$digest(in_hashes) 21 | 22 | in_hash 23 | }, overwrite = TRUE) 24 | 25 | 26 | # initialize ---- 27 | R6FileSource$set("public", "initialize", function( 28 | fn, 29 | fn_key, 30 | fn_name, 31 | fn_id, 32 | flow_options = get_flow_options() 33 | ) { 34 | super$initialize(fn, fn_key, fn_name, fn_id, flow_options) 35 | 36 | # after registering into eddy, remove itself if error 37 | tryCatch({ 38 | # calc_in_hash 39 | self$calc_in_hash <- self$calc_in_hash_file_source 40 | }, error = function(e) { 41 | self$eddy$remove_flow(fn_key) 42 | stop(e) 43 | }) 44 | 45 | invisible(NULL) 46 | }, overwrite = TRUE) 47 | 48 | 49 | #' Creates a flow object that watches one or more files 50 | #' 51 | #' @details 52 | #' This flow object does not throw an error if the \code{file_path} is missing, 53 | #' but it changes its state. Hence, it can be used to trigger a downstream 54 | #' flow object if the file is now present, changed or missing. 55 | #' 56 | #' @param file_path A (named) vector of file paths to be watched or a 57 | #' \code{fs_path} object. 58 | #' @param flow_options List of options created using \code{get_flow_options}. 59 | #' All options except \code{eddy} are ignored. 60 | #' 61 | #' @return The flow object. 62 | #' 63 | #' @examples 64 | #' # write for the first time content in file and create flow 65 | #' file_temp <- tempfile(pattern = "example_source") 66 | #' write.csv(head(mtcars), file_temp, row.names = FALSE) 67 | #' rflow_source <- flow_file_source(file_temp) 68 | #' 69 | #' # write other content in the same file 70 | #' # now the flow object will update its state 71 | #' write.csv(tail(mtcars), file_temp, row.names = FALSE) 72 | #' rflow_source <- flow_file_source(file_temp) 73 | #' unlink(file_temp) 74 | #' 75 | #' @export 76 | flow_file_source <- function(file_path, 77 | flow_options = get_flow_options()) { 78 | 79 | # validate file_path 80 | stopifnot(is.character(file_path)) 81 | stopifnot(length(file_path) >= 1L) 82 | nms <- names(file_path) 83 | if (is.null(nms)) { 84 | names(file_path) <- file_path 85 | } else { 86 | nms[nms == ""] <- file_path[nms == ""] 87 | names(file_path) <- nms 88 | } 89 | lst <- as.list(file_path) 90 | if (!rlang::is_dictionaryish(lst)) { 91 | rlang::abort(paste( 92 | "Invalid/Duplicate names detected;", 93 | "please provide a vector with unique names.")) 94 | } 95 | 96 | flow_options$excluded_arg <- character() 97 | flow_options$eval_arg_fn <- NULL 98 | flow_options$split_bare_list <- TRUE 99 | flow_options$split_dataframe <- FALSE 100 | flow_options$split_fn <- as.list 101 | eddy <- flow_options$eddy 102 | 103 | # fn_name based on file_path 104 | fn_name <- if (length(file_path) > 1L) { 105 | paste0(file_path[1L], " [+", length(file_path) - 1L, "]") 106 | } else { 107 | file_path 108 | } 109 | nchar_max <- 30L 110 | if (nchar(fn_name) > nchar_max) { 111 | nchar_side <- nchar_max %/% 2L - 2L 112 | n1 <- substr(fn_name, 1L, nchar_side) 113 | n2 <- substr(fn_name, nchar(fn_name) - nchar_side, nchar(fn_name)) 114 | fn_name <- paste0(n1, "...", n2) 115 | } 116 | 117 | # fn_id is always the same for each path set, cannot be set by user 118 | fn_id <- 1L 119 | 120 | # cannot use make_key(), fn_key depends only on path string 121 | fn_key <- eddy$digest(file_path) 122 | 123 | if (eddy$has_flow(fn_key)) { 124 | flow <- eddy$get_flow(fn_key) 125 | } else { 126 | flow <- R6FileSource$new( 127 | fn = identity, 128 | fn_key = fn_key, 129 | fn_name = fn_name, 130 | fn_id = fn_id, 131 | flow_options = flow_options 132 | ) 133 | } 134 | 135 | # eval the call 136 | do.call( 137 | what = flow$rf_fn, 138 | args = list(x = file_path), 139 | envir = parent.frame(n = 2) 140 | ) 141 | } 142 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rflow - Flexible R Pipelines with Caching 2 | 3 | [![Travis build status](https://travis-ci.org/numeract/rflow.svg?branch=master)](https://travis-ci.org/numeract/rflow) 4 | [![Coverage status](https://codecov.io/gh/numeract/rflow/branch/master/graph/badge.svg)](https://codecov.io/github/numeract/rflow?branch=master) 5 | [![CRAN status](https://www.r-pkg.org/badges/version/rflow)](https://cran.r-project.org/package=rflow) 6 | [![lifecycle](https://img.shields.io/badge/lifecycle-maturing-blue.svg)](https://www.tidyverse.org/lifecycle/#maturing) 7 | 8 | **The package is currently under active development, please expect major 9 | changes while the API stabilizes.** 10 | 11 | 12 | ## Motivation 13 | 14 | A common problem when processing data as part of a pipeline is avoiding 15 | unnecessary calculations. For example, if a function is called over and 16 | over with the same arguments, it should not recalculate the result each time 17 | but it should provide the cached (pre-computed) result. 18 | 19 | While caching of the function output resolves the first problem, a second 20 | issue occurs when large data sets are being processed. In this case, hashing 21 | of the input arguments each time might take too long. This issue can be solved 22 | by hashing the data only once (as output) and then by noticing changes 23 | in the hash received by the downstream function. In other words, it is not 24 | the data that flows through the pipeline (as is the case with standard function), 25 | but hashes of the data. 26 | 27 | A third issue is output sub-setting. When working with a pipeline there is 28 | often the case (e.g. ETL, Machine Learning) that we need to pass the whole 29 | data frame but the function is going to use only a subset (e.g. a CV fold). 30 | Since the main data frame has changes, caching of the result is no longer 31 | efficient. The solution involves hashing of the subset of interest which 32 | can be done by introducing additional intermediate functions in the pipeline. 33 | However, there is a loss of efficiency due to excessive rehashing as the 34 | main data frame passes through many functions. 35 | 36 | The package `rflow` addresses these inefficiencies and makes pipelines as easy 37 | to use as in tidyverse. 38 | 39 | 40 | ## Installation 41 | 42 | ``` 43 | # install.packages("devtools") 44 | devtools::install_github("numeract/rflow") 45 | ``` 46 | 47 | 48 | ## Use 49 | 50 | 51 | ### Simple Example 52 | 53 | ``` 54 | x1 <- 10 55 | x2 <- 0.5 56 | x3 <- 2 57 | 58 | f1 <- function(a, b, c = 1) {a * b + c} 59 | f2 <- function(d, e) {d / e} 60 | 61 | # passing the results downstream using functions 62 | (o1 <- f1(x1, x2)) # 6 63 | (o2 <- f2(o1, x3)) # 3 64 | 65 | 66 | # variant 1: declaring flows for each function using default options 67 | ff1 <- make_flow_fn(f1) 68 | ff2 <- make_flow_fn(f2) 69 | 70 | # passing to the downstream flow and collecting the results 71 | r1 <- ff1(x1, x2) # does not trigger re-calc 72 | r2 <- ff2(r1, x3) # does not trigger re-calc; first arg. is a flow arg. 73 | collect(r1) # 6 74 | collect(r2) # 3 75 | 76 | 77 | # variant 2: arguments and functions withing one call 78 | library(dplyr) # makes life easier 79 | flow_fn(x1, x2, fn = f1) %>% # reuses cache created by ff1 80 | flow_fn(x3, fn = f2) %>% # reuses cache created by ff2 81 | collect() # 3, no actual re-calc takes place 82 | ``` 83 | 84 | 85 | ### Pipelines 86 | 87 | 1. Create your function, e.g. `f <- function(...) {...}` 88 | - `rflow` works best with pure functions, i.e. functions 89 | that depend only on their inputs (and not on variables outside the function 90 | frame) and do not produce any side effects (e.g. printing, modifying variables 91 | in the global environment). 92 | 93 | 2. "flow" the function: `ff <- make_flow_fn(f))` 94 | 95 | 3. When pipelining `ff` into another `rflow` function, simply supply `ff()` 96 | as an argument, for example: `ff(x) %>% ff2(y) %>% ff3(z)` 97 | 98 | 4. At the end of the `rflow` pipeline you must use `collect()` to collect 99 | the actual data (and not just the cached structure). Alternatively, 100 | use `flow_ns_sink()` to dump the data into an environment or a 101 | `Shiny::reactiveValues` name space. 102 | 103 | 104 | ### Shiny 105 | 106 | Shiny from RStudio uses reactive values to know what changes took place and 107 | what to recompute. It is thus possible to use a series of reactive elements 108 | in Shiny to prevent expensive re-computations from taking place. Example: 109 | 110 | ``` 111 | rv1 <- reactive({ 112 | ... input$x ... 113 | }) 114 | 115 | rv2 <- reactive({ 116 | ... rv1() .... input$y ... 117 | }) 118 | 119 | rv3 <- reactive({ 120 | ... rv2() .... input$z ... 121 | }) 122 | ``` 123 | 124 | The downside is that we need one reactive element for each function in the 125 | pipeline - this makes data processing dependent on UI / Shiny. Using `rflow`, 126 | we can separate the UI from the data processing, maintaining the caching 127 | not only for the current state but for all previously computed states. 128 | 129 | ``` 130 | rv <- reactive({ 131 | rf1(input$x, ...) %>% 132 | rf2(input$y, ...) %>% 133 | rf3(input$z, ...) %>% 134 | collect() 135 | }) 136 | ``` 137 | 138 | While a similar workflow can be achieved with package `memoise`, it suffers from 139 | several disadvantages (below). 140 | 141 | 142 | ### Output Subset 143 | 144 | (to be updated) 145 | 146 | 147 | ## Other frameworks 148 | 149 | 150 | ### Memoise 151 | 152 | Package [memoise](https://github.com/r-lib/memoise) 153 | by Hadley Wickham, Jim Hester and others was the main source of inspiration. 154 | Memoise is elegant, fast, simple to use, but it suffers from certain limitations 155 | that we hope to overcome in this package: 156 | 157 | - excessive [rehashing of inputs](https://github.com/r-lib/memoise/issues/31) 158 | - only one cache layer (although its cache framework is extensible) 159 | - no input/output sub-setting, it uses the complete set of arguments provided 160 | - no reactivity (yet to be implemented in `rflow`) 161 | 162 | 163 | ### Drake 164 | 165 | Package [drake](https://github.com/ropensci/drake) by Will Landau and others 166 | provides a complete framework for large data sets, including using 167 | files as inputs and outputs. The downside is that it requires additional 168 | overhead to get started and its focus is on the pipeline as a whole. If your 169 | work requires many hours of computations (which increases the value of each 170 | result), the overhead due to the setup has a relatively lower cost - in this 171 | scenario `drake` is an excellent choice. 172 | 173 | Package `rflow` is somewhere between `memoise` and `drake`: 174 | 175 | - one can start using `rflow` right away, with minimal overhead 176 | - allows focusing on the data processing (e.g., EDA) and not on the framework 177 | 178 | 179 | ## TODO list 180 | 181 | - reactivity 182 | - multi-layer cache (with file locking) 183 | - files sinks 184 | - parallel processing 185 | -------------------------------------------------------------------------------- /_pkgdown.yml: -------------------------------------------------------------------------------- 1 | reference: 2 | - title: Flow Functions 3 | contents: 4 | - starts_with("flow_") 5 | - make_flow_fn 6 | - collect 7 | - element 8 | - is_flow 9 | - is_flow_fn 10 | - is_not_flow_fn 11 | - is_current 12 | - is_valid 13 | - title: Eddy Functions 14 | contents: 15 | - ends_with("eddy") 16 | - title: Cache Functions 17 | contents: 18 | - starts_with("cache_") 19 | - ends_with("cache") 20 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false 2 | 3 | coverage: 4 | status: 5 | project: 6 | default: 7 | target: auto 8 | threshold: 1% 9 | patch: 10 | default: 11 | target: auto 12 | threshold: 1% 13 | 14 | ignore: 15 | - "**/nmisc.R" 16 | -------------------------------------------------------------------------------- /cran-comments.md: -------------------------------------------------------------------------------- 1 | ## Test environments 2 | 3 | - ? 4 | 5 | ## R CMD check results 6 | 7 | - ? 8 | - this is the first submission 9 | 10 | 11 | ## Downstream dependencies 12 | 13 | - no downstream dependencies on CRAN 14 | 15 | 16 | ## Resubmit comments 17 | 18 | - this is the first submission 19 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numeract/rflow/858cb72ea6bd1230870f75240746406b40e1000e/docs/.nojekyll -------------------------------------------------------------------------------- /docs/LICENSE-text.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | License • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 44 | 45 | 46 | 47 | 48 | 49 |
50 |
51 | 105 | 106 | 107 |
108 | 109 |
110 |
111 | 114 | 115 |
MIT License
116 | 
117 | Copyright (c) 2018 Numeract
118 | 
119 | Permission is hereby granted, free of charge, to any person obtaining a copy
120 | of this software and associated documentation files (the "Software"), to deal
121 | in the Software without restriction, including without limitation the rights
122 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
123 | copies of the Software, and to permit persons to whom the Software is
124 | furnished to do so, subject to the following conditions:
125 | 
126 | The above copyright notice and this permission notice shall be included in all
127 | copies or substantial portions of the Software.
128 | 
129 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
130 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
131 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
132 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
133 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
134 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
135 | SOFTWARE.
136 | 
137 | 138 |
139 | 140 |
141 | 142 | 143 | 153 |
154 | 155 | 156 | 157 | 158 | 159 | 160 | -------------------------------------------------------------------------------- /docs/articles/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Articles • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 44 | 45 | 46 | 47 | 48 | 49 |
50 |
51 | 105 | 106 | 107 |
108 | 109 |
110 |
111 | 114 | 115 |
116 |

All vignettes

117 |

118 | 119 | 122 |
123 |
124 |
125 | 126 | 136 |
137 | 138 | 139 | 140 | 141 | 142 | 143 | -------------------------------------------------------------------------------- /docs/authors.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Authors • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 44 | 45 | 46 | 47 | 48 | 49 |
50 |
51 | 105 | 106 | 107 |
108 | 109 |
110 |
111 | 114 | 115 |
    116 |
  • 117 |

    Mike Badescu. Author, maintainer. 118 |

    119 |
  • 120 |
  • 121 |

    Ana-Maria Niculescu. Author. 122 |

    123 |
  • 124 |
  • 125 |

    Teodor Ciuraru. Author. 126 |

    127 |
  • 128 |
  • 129 |

    Numeract LLC. Copyright holder. 130 |

    131 |
  • 132 |
133 | 134 |
135 | 136 |
137 | 138 | 139 | 149 |
150 | 151 | 152 | 153 | 154 | 155 | 156 | -------------------------------------------------------------------------------- /docs/docsearch.js: -------------------------------------------------------------------------------- 1 | $(function() { 2 | 3 | // register a handler to move the focus to the search bar 4 | // upon pressing shift + "/" (i.e. "?") 5 | $(document).on('keydown', function(e) { 6 | if (e.shiftKey && e.keyCode == 191) { 7 | e.preventDefault(); 8 | $("#search-input").focus(); 9 | } 10 | }); 11 | 12 | $(document).ready(function() { 13 | // do keyword highlighting 14 | /* modified from https://jsfiddle.net/julmot/bL6bb5oo/ */ 15 | var mark = function() { 16 | 17 | var referrer = document.URL ; 18 | var paramKey = "q" ; 19 | 20 | if (referrer.indexOf("?") !== -1) { 21 | var qs = referrer.substr(referrer.indexOf('?') + 1); 22 | var qs_noanchor = qs.split('#')[0]; 23 | var qsa = qs_noanchor.split('&'); 24 | var keyword = ""; 25 | 26 | for (var i = 0; i < qsa.length; i++) { 27 | var currentParam = qsa[i].split('='); 28 | 29 | if (currentParam.length !== 2) { 30 | continue; 31 | } 32 | 33 | if (currentParam[0] == paramKey) { 34 | keyword = decodeURIComponent(currentParam[1].replace(/\+/g, "%20")); 35 | } 36 | } 37 | 38 | if (keyword !== "") { 39 | $(".contents").unmark({ 40 | done: function() { 41 | $(".contents").mark(keyword); 42 | } 43 | }); 44 | } 45 | } 46 | }; 47 | 48 | mark(); 49 | }); 50 | }); 51 | 52 | /* Search term highlighting ------------------------------*/ 53 | 54 | function matchedWords(hit) { 55 | var words = []; 56 | 57 | var hierarchy = hit._highlightResult.hierarchy; 58 | // loop to fetch from lvl0, lvl1, etc. 59 | for (var idx in hierarchy) { 60 | words = words.concat(hierarchy[idx].matchedWords); 61 | } 62 | 63 | var content = hit._highlightResult.content; 64 | if (content) { 65 | words = words.concat(content.matchedWords); 66 | } 67 | 68 | // return unique words 69 | var words_uniq = [...new Set(words)]; 70 | return words_uniq; 71 | } 72 | 73 | function updateHitURL(hit) { 74 | 75 | var words = matchedWords(hit); 76 | var url = ""; 77 | 78 | if (hit.anchor) { 79 | url = hit.url_without_anchor + '?q=' + escape(words.join(" ")) + '#' + hit.anchor; 80 | } else { 81 | url = hit.url + '?q=' + escape(words.join(" ")); 82 | } 83 | 84 | return url; 85 | } 86 | -------------------------------------------------------------------------------- /docs/link.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 8 | 12 | 13 | -------------------------------------------------------------------------------- /docs/pkgdown.css: -------------------------------------------------------------------------------- 1 | /* Sticky footer */ 2 | 3 | /** 4 | * Basic idea: https://philipwalton.github.io/solved-by-flexbox/demos/sticky-footer/ 5 | * Details: https://github.com/philipwalton/solved-by-flexbox/blob/master/assets/css/components/site.css 6 | * 7 | * .Site -> body > .container 8 | * .Site-content -> body > .container .row 9 | * .footer -> footer 10 | * 11 | * Key idea seems to be to ensure that .container and __all its parents__ 12 | * have height set to 100% 13 | * 14 | */ 15 | 16 | html, body { 17 | height: 100%; 18 | } 19 | 20 | body > .container { 21 | display: flex; 22 | height: 100%; 23 | flex-direction: column; 24 | 25 | padding-top: 60px; 26 | } 27 | 28 | body > .container .row { 29 | flex: 1 0 auto; 30 | } 31 | 32 | footer { 33 | margin-top: 45px; 34 | padding: 35px 0 36px; 35 | border-top: 1px solid #e5e5e5; 36 | color: #666; 37 | display: flex; 38 | flex-shrink: 0; 39 | } 40 | footer p { 41 | margin-bottom: 0; 42 | } 43 | footer div { 44 | flex: 1; 45 | } 46 | footer .pkgdown { 47 | text-align: right; 48 | } 49 | footer p { 50 | margin-bottom: 0; 51 | } 52 | 53 | img.icon { 54 | float: right; 55 | } 56 | 57 | img { 58 | max-width: 100%; 59 | } 60 | 61 | /* Typographic tweaking ---------------------------------*/ 62 | 63 | .contents h1.page-header { 64 | margin-top: calc(-60px + 1em); 65 | } 66 | 67 | /* Section anchors ---------------------------------*/ 68 | 69 | a.anchor { 70 | margin-left: -30px; 71 | display:inline-block; 72 | width: 30px; 73 | height: 30px; 74 | visibility: hidden; 75 | 76 | background-image: url(./link.svg); 77 | background-repeat: no-repeat; 78 | background-size: 20px 20px; 79 | background-position: center center; 80 | } 81 | 82 | .hasAnchor:hover a.anchor { 83 | visibility: visible; 84 | } 85 | 86 | @media (max-width: 767px) { 87 | .hasAnchor:hover a.anchor { 88 | visibility: hidden; 89 | } 90 | } 91 | 92 | 93 | /* Fixes for fixed navbar --------------------------*/ 94 | 95 | .contents h1, .contents h2, .contents h3, .contents h4 { 96 | padding-top: 60px; 97 | margin-top: -40px; 98 | } 99 | 100 | /* Static header placement on mobile devices */ 101 | @media (max-width: 767px) { 102 | .navbar-fixed-top { 103 | position: absolute; 104 | } 105 | .navbar { 106 | padding: 0; 107 | } 108 | } 109 | 110 | 111 | /* Sidebar --------------------------*/ 112 | 113 | #sidebar { 114 | margin-top: 30px; 115 | } 116 | #sidebar h2 { 117 | font-size: 1.5em; 118 | margin-top: 1em; 119 | } 120 | 121 | #sidebar h2:first-child { 122 | margin-top: 0; 123 | } 124 | 125 | #sidebar .list-unstyled li { 126 | margin-bottom: 0.5em; 127 | } 128 | 129 | .orcid { 130 | height: 16px; 131 | vertical-align: middle; 132 | } 133 | 134 | /* Reference index & topics ----------------------------------------------- */ 135 | 136 | .ref-index th {font-weight: normal;} 137 | 138 | .ref-index td {vertical-align: top;} 139 | .ref-index .alias {width: 40%;} 140 | .ref-index .title {width: 60%;} 141 | 142 | .ref-index .alias {width: 40%;} 143 | .ref-index .title {width: 60%;} 144 | 145 | .ref-arguments th {text-align: right; padding-right: 10px;} 146 | .ref-arguments th, .ref-arguments td {vertical-align: top;} 147 | .ref-arguments .name {width: 20%;} 148 | .ref-arguments .desc {width: 80%;} 149 | 150 | /* Nice scrolling for wide elements --------------------------------------- */ 151 | 152 | table { 153 | display: block; 154 | overflow: auto; 155 | } 156 | 157 | /* Syntax highlighting ---------------------------------------------------- */ 158 | 159 | pre { 160 | word-wrap: normal; 161 | word-break: normal; 162 | border: 1px solid #eee; 163 | } 164 | 165 | pre, code { 166 | background-color: #f8f8f8; 167 | color: #333; 168 | } 169 | 170 | pre code { 171 | overflow: auto; 172 | word-wrap: normal; 173 | white-space: pre; 174 | } 175 | 176 | pre .img { 177 | margin: 5px 0; 178 | } 179 | 180 | pre .img img { 181 | background-color: #fff; 182 | display: block; 183 | height: auto; 184 | } 185 | 186 | code a, pre a { 187 | color: #375f84; 188 | } 189 | 190 | a.sourceLine:hover { 191 | text-decoration: none; 192 | } 193 | 194 | .fl {color: #1514b5;} 195 | .fu {color: #000000;} /* function */ 196 | .ch,.st {color: #036a07;} /* string */ 197 | .kw {color: #264D66;} /* keyword */ 198 | .co {color: #888888;} /* comment */ 199 | 200 | .message { color: black; font-weight: bolder;} 201 | .error { color: orange; font-weight: bolder;} 202 | .warning { color: #6A0366; font-weight: bolder;} 203 | 204 | /* Clipboard --------------------------*/ 205 | 206 | .hasCopyButton { 207 | position: relative; 208 | } 209 | 210 | .btn-copy-ex { 211 | position: absolute; 212 | right: 0; 213 | top: 0; 214 | visibility: hidden; 215 | } 216 | 217 | .hasCopyButton:hover button.btn-copy-ex { 218 | visibility: visible; 219 | } 220 | 221 | /* mark.js ----------------------------*/ 222 | 223 | mark { 224 | background-color: rgba(255, 255, 51, 0.5); 225 | border-bottom: 2px solid rgba(255, 153, 51, 0.3); 226 | padding: 1px; 227 | } 228 | 229 | /* vertical spacing after htmlwidgets */ 230 | .html-widget { 231 | margin-bottom: 10px; 232 | } 233 | -------------------------------------------------------------------------------- /docs/pkgdown.js: -------------------------------------------------------------------------------- 1 | /* http://gregfranko.com/blog/jquery-best-practices/ */ 2 | (function($) { 3 | $(function() { 4 | 5 | $("#sidebar") 6 | .stick_in_parent({offset_top: 40}) 7 | .on('sticky_kit:bottom', function(e) { 8 | $(this).parent().css('position', 'static'); 9 | }) 10 | .on('sticky_kit:unbottom', function(e) { 11 | $(this).parent().css('position', 'relative'); 12 | }); 13 | 14 | $('body').scrollspy({ 15 | target: '#sidebar', 16 | offset: 60 17 | }); 18 | 19 | $('[data-toggle="tooltip"]').tooltip(); 20 | 21 | var cur_path = paths(location.pathname); 22 | var links = $("#navbar ul li a"); 23 | var max_length = -1; 24 | var pos = -1; 25 | for (var i = 0; i < links.length; i++) { 26 | if (links[i].getAttribute("href") === "#") 27 | continue; 28 | var path = paths(links[i].pathname); 29 | 30 | var length = prefix_length(cur_path, path); 31 | if (length > max_length) { 32 | max_length = length; 33 | pos = i; 34 | } 35 | } 36 | 37 | // Add class to parent
  • , and enclosing
  • if in dropdown 38 | if (pos >= 0) { 39 | var menu_anchor = $(links[pos]); 40 | menu_anchor.parent().addClass("active"); 41 | menu_anchor.closest("li.dropdown").addClass("active"); 42 | } 43 | }); 44 | 45 | function paths(pathname) { 46 | var pieces = pathname.split("/"); 47 | pieces.shift(); // always starts with / 48 | 49 | var end = pieces[pieces.length - 1]; 50 | if (end === "index.html" || end === "") 51 | pieces.pop(); 52 | return(pieces); 53 | } 54 | 55 | function prefix_length(needle, haystack) { 56 | if (needle.length > haystack.length) 57 | return(0); 58 | 59 | // Special case for length-0 haystack, since for loop won't run 60 | if (haystack.length === 0) { 61 | return(needle.length === 0 ? 1 : 0); 62 | } 63 | 64 | for (var i = 0; i < haystack.length; i++) { 65 | if (needle[i] != haystack[i]) 66 | return(i); 67 | } 68 | 69 | return(haystack.length); 70 | } 71 | 72 | /* Clipboard --------------------------*/ 73 | 74 | function changeTooltipMessage(element, msg) { 75 | var tooltipOriginalTitle=element.getAttribute('data-original-title'); 76 | element.setAttribute('data-original-title', msg); 77 | $(element).tooltip('show'); 78 | element.setAttribute('data-original-title', tooltipOriginalTitle); 79 | } 80 | 81 | if(Clipboard.isSupported()) { 82 | $(document).ready(function() { 83 | var copyButton = ""; 84 | 85 | $(".examples, div.sourceCode").addClass("hasCopyButton"); 86 | 87 | // Insert copy buttons: 88 | $(copyButton).prependTo(".hasCopyButton"); 89 | 90 | // Initialize tooltips: 91 | $('.btn-copy-ex').tooltip({container: 'body'}); 92 | 93 | // Initialize clipboard: 94 | var clipboardBtnCopies = new Clipboard('[data-clipboard-copy]', { 95 | text: function(trigger) { 96 | return trigger.parentNode.textContent; 97 | } 98 | }); 99 | 100 | clipboardBtnCopies.on('success', function(e) { 101 | changeTooltipMessage(e.trigger, 'Copied!'); 102 | e.clearSelection(); 103 | }); 104 | 105 | clipboardBtnCopies.on('error', function() { 106 | changeTooltipMessage(e.trigger,'Press Ctrl+C or Command+C to copy'); 107 | }); 108 | }); 109 | } 110 | })(window.jQuery || window.$) 111 | -------------------------------------------------------------------------------- /docs/pkgdown.yml: -------------------------------------------------------------------------------- 1 | pandoc: 1.19.2.1 2 | pkgdown: 1.1.0.9000 3 | pkgdown_sha: 6ca8a64d7867469d65e17e2ab1f9686f501c3c06 4 | articles: 5 | rflow-vignette: rflow-vignette.html 6 | 7 | -------------------------------------------------------------------------------- /docs/reference/cache_file.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Get the file only cache engine. — cache_file • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 108 | 109 | 110 |
    111 | 112 |
    113 |
    114 | 119 | 120 |
    121 | 122 |

    All cache data will be stored only on the local disk.

    123 | 124 |
    125 | 126 |
    cache_file(cache_dir)
    127 | 128 |

    Arguments

    129 | 130 | 131 | 132 | 133 | 134 | 135 |
    cache_dir

    Directory where to store the cache files.

    136 | 137 |

    Value

    138 | 139 |

    A cache object that inherits from R6Cache.

    140 | 141 |

    See also

    142 | 143 |

    Other cache functions: cache_memory_file, 144 | cache_memory, default_cache

    145 | 146 | 147 |

    Examples

    148 |
    # NOT RUN {
    149 | cache_in_file <- cache_file("path_to_cache_diretory")
    150 | # }
    151 |
    152 |
    153 | 166 |
    167 | 168 |
    169 | 172 | 173 |
    174 |

    Site built with pkgdown.

    175 |
    176 | 177 |
    178 |
    179 | 180 | 181 | 182 | 183 | 184 | 185 | -------------------------------------------------------------------------------- /docs/reference/cache_memory.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Get the memory only cache engine. — cache_memory • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 108 | 109 | 110 |
    111 | 112 |
    113 |
    114 | 119 | 120 |
    121 | 122 |

    All cache data will be stored only in memory.

    123 | 124 |
    125 | 126 |
    cache_memory()
    127 | 128 |

    Value

    129 | 130 |

    A cache object that inherits from R6Cache.

    131 | 132 |

    See also

    133 | 134 |

    Other cache functions: cache_file, 135 | cache_memory_file, 136 | default_cache

    137 | 138 | 139 |

    Examples

    140 |
    cache_in_memory <- cache_memory()
    141 |
    142 | 154 |
    155 | 156 |
    157 | 160 | 161 |
    162 |

    Site built with pkgdown.

    163 |
    164 | 165 |
    166 |
    167 | 168 | 169 | 170 | 171 | 172 | 173 | -------------------------------------------------------------------------------- /docs/reference/cache_memory_file.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Get the memory-file cache engine. — cache_memory_file • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 48 | 49 | 50 | 51 | 52 | 53 |
    54 |
    55 | 109 | 110 | 111 |
    112 | 113 |
    114 |
    115 | 120 | 121 |
    122 | 123 |

    The cache data will be stored both in memory (for quick access) and on 124 | disk (for persistence).

    125 | 126 |
    127 | 128 |
    cache_memory_file(cache_dir)
    129 | 130 |

    Arguments

    131 | 132 | 133 | 134 | 135 | 136 | 137 |
    cache_dir

    Directory where to store the cache files.

    138 | 139 |

    Value

    140 | 141 |

    A cache object that inherits from R6Cache.

    142 | 143 |

    See also

    144 | 145 |

    Other cache functions: cache_file, 146 | cache_memory, default_cache

    147 | 148 | 149 |

    Examples

    150 |
    # NOT RUN {
    151 | cache_fmem <- cache_memory_file("path_to_cache_diretory")
    152 | # }
    153 | 154 |
    155 |
    156 | 169 |
    170 | 171 |
    172 | 175 | 176 |
    177 |

    Site built with pkgdown.

    178 |
    179 | 180 |
    181 |
    182 | 183 | 184 | 185 | 186 | 187 | 188 | -------------------------------------------------------------------------------- /docs/reference/compute.R6Flow.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Trigger computation for an <code>R6Flow</code> or an <code>Element</code> object. — compute • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 108 | 109 | 110 |
    111 | 112 |
    113 |
    114 | 119 | 120 |
    121 | 122 |

    Trigger computation for an R6Flow or an Element object.

    123 | 124 |
    125 | 126 |
    # S3 method for R6Flow
    127 | compute(x, ...)
    128 | 
    129 | # S3 method for Element
    130 | compute(x, ...)
    131 | 132 |

    Arguments

    133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 |
    x

    A flow object, e.g. as returned by flow_fn.

    ...

    Any other arguments will be ignored.

    144 | 145 |

    Value

    146 | 147 |

    Logical, whether the result is available to be collected.

    148 | 149 |

    Details

    150 | 151 |

    Unlike collect, it does not trigger an error if it fails 152 | to compute and it does not return the actual result of the computation.

    153 | 154 | 155 |
    156 | 167 |
    168 | 169 |
    170 | 173 | 174 |
    175 |

    Site built with pkgdown.

    176 |
    177 | 178 |
    179 |
    180 | 181 | 182 | 183 | 184 | 185 | 186 | -------------------------------------------------------------------------------- /docs/reference/default_cache.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Get the default cache engine. — default_cache • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 48 | 49 | 50 | 51 | 52 | 53 |
    54 |
    55 | 109 | 110 | 111 |
    112 | 113 |
    114 |
    115 | 120 | 121 |
    122 | 123 |

    Currently, the default cache engine is cache_memory(), but this 124 | might change in the future.

    125 | 126 |
    127 | 128 |
    default_cache()
    129 | 130 |

    Value

    131 | 132 |

    A cache object that inherits from R6Cache.

    133 | 134 |

    See also

    135 | 136 |

    Other cache functions: cache_file, 137 | cache_memory_file, 138 | cache_memory

    139 | 140 | 141 |

    Examples

    142 |
    # NOT RUN {
    143 | current_cache <- default_cache()
    144 | # }
    145 |
    146 |
    147 | 159 |
    160 | 161 |
    162 | 165 | 166 |
    167 |

    Site built with pkgdown.

    168 |
    169 | 170 |
    171 |
    172 | 173 | 174 | 175 | 176 | 177 | 178 | -------------------------------------------------------------------------------- /docs/reference/default_eddy_env.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Get the default binding environment that keeps the eddies. — default_eddy_env • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 108 | 109 | 110 |
    111 | 112 |
    113 |
    114 | 119 | 120 |
    121 | 122 |

    Get the default binding environment that keeps the eddies.

    123 | 124 |
    125 | 126 |
    default_eddy_env()
    127 | 128 |

    Value

    129 | 130 |

    An environment.

    131 | 132 | 133 |
    134 | 142 |
    143 | 144 |
    145 | 148 | 149 |
    150 |

    Site built with pkgdown.

    151 |
    152 | 153 |
    154 |
    155 | 156 | 157 | 158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /docs/reference/imports.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Imports from other packages — %>% • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 108 | 109 | 110 |
    111 | 112 |
    113 |
    114 | 119 | 120 |
    121 | 122 |

    Imports from other packages

    123 | 124 |
    125 | 126 | 127 | 128 |
    129 | 135 |
    136 | 137 |
    138 | 141 | 142 |
    143 |

    Site built with pkgdown.

    144 |
    145 | 146 |
    147 |
    148 | 149 | 150 | 151 | 152 | 153 | 154 | -------------------------------------------------------------------------------- /docs/reference/is_current.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Does the flow have a "current" state? — is_current • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 49 | 50 | 51 | 52 | 53 | 54 |
    55 |
    56 | 110 | 111 | 112 |
    113 | 114 |
    115 |
    116 | 121 | 122 |
    123 | 124 |

    If there is no current state, e.g. right after make_flow_fn, 125 | the flow is "not flowing", it is preventing downstream flows 126 | from being computed.

    127 | 128 |
    129 | 130 |
    is_current(flow)
    131 | 132 |

    Arguments

    133 | 134 | 135 | 136 | 137 | 138 | 139 |
    flow

    A flow object, e.g. as returned by flow_fn.

    140 | 141 |

    Value

    142 | 143 |

    A logical value, whether the current state is valid.

    144 | 145 | 146 |

    Examples

    147 |
    fn <- function(x, y) { x + y + 8 } 148 | flowed_fn <- flow_fn(1, 2, fn = fn)
    #> New cache: fn=fn / fn_id=1 / fn_key=f44b017eab837ef5
    is_current_flow <- is_current(flowed_fn)
    149 |
    150 | 161 |
    162 | 163 |
    164 | 167 | 168 |
    169 |

    Site built with pkgdown.

    170 |
    171 | 172 |
    173 |
    174 | 175 | 176 | 177 | 178 | 179 | 180 | -------------------------------------------------------------------------------- /docs/reference/is_flow.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Is the object a flow object or a flow element? — is_flow • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 108 | 109 | 110 |
    111 | 112 |
    113 |
    114 | 119 | 120 |
    121 | 122 |

    Is the object a flow object or a flow element?

    123 | 124 |
    125 | 126 |
    is_flow(x)
    127 | 128 |

    Arguments

    129 | 130 | 131 | 132 | 133 | 134 | 135 |
    x

    An object.

    136 | 137 |

    Value

    138 | 139 |

    A logical value, whether x is a flow object.

    140 | 141 | 142 |

    Examples

    143 |
    fn <- function(x, y) { x + y + 11 } 144 | flow_function <- flow_fn(1, 2, fn = fn)
    #> New cache: fn=fn / fn_id=1 / fn_key=a738cc162753ad15
    is_input_flow <- is_flow(flow_function)
    145 |
    146 | 157 |
    158 | 159 |
    160 | 163 | 164 |
    165 |

    Site built with pkgdown.

    166 |
    167 | 168 |
    169 |
    170 | 171 | 172 | 173 | 174 | 175 | 176 | -------------------------------------------------------------------------------- /docs/reference/is_flow_fn.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Is the function a flow function (as returned by <code>make_flow_fn</code>)? — is_flow_fn • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 108 | 109 | 110 |
    111 | 112 |
    113 |
    114 | 119 | 120 |
    121 | 122 |

    Is the function a flow function (as returned by make_flow_fn)?

    123 | 124 |
    125 | 126 |
    is_flow_fn(fn)
    127 | 128 |

    Arguments

    129 | 130 | 131 | 132 | 133 | 134 | 135 |
    fn

    A function.

    136 | 137 |

    Value

    138 | 139 |

    A logical value, whether fn is a flow function.

    140 | 141 | 142 |

    Examples

    143 |
    fn <- function(x, y) { x + y + 12 } 144 | flowed_function <- flow_fn(2, 3, fn = fn)
    #> New cache: fn=fn / fn_id=1 / fn_key=9ea087ad94ba4d61
    is_flow_function <- is_flow_fn(fn = flowed_function)
    145 |
    146 | 157 |
    158 | 159 |
    160 | 163 | 164 |
    165 |

    Site built with pkgdown.

    166 |
    167 | 168 |
    169 |
    170 | 171 | 172 | 173 | 174 | 175 | 176 | -------------------------------------------------------------------------------- /docs/reference/is_not_flow_fn.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Check if the function is NOT a flow function 10 | (as returned by <code>make_flow_fn</code>). — is_not_flow_fn • rflow 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 36 | 37 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 50 | 51 | 52 | 53 | 54 | 55 |
    56 |
    57 | 111 | 112 | 113 |
    114 | 115 |
    116 |
    117 | 123 | 124 |
    125 | 126 |

    Check if the function is NOT a flow function 127 | (as returned by make_flow_fn).

    128 | 129 |
    130 | 131 |
    is_not_flow_fn(fn)
    132 | 133 |

    Arguments

    134 | 135 | 136 | 137 | 138 | 139 | 140 |
    fn

    A function.

    141 | 142 |

    Value

    143 | 144 |

    A logical value, whether fn is a not flow function.

    145 | 146 | 147 |
    148 | 157 |
    158 | 159 |
    160 | 163 | 164 |
    165 |

    Site built with pkgdown.

    166 |
    167 | 168 |
    169 |
    170 | 171 | 172 | 173 | 174 | 175 | 176 | -------------------------------------------------------------------------------- /docs/reference/is_valid.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Is the current state valid (stored in the cache)? — is_valid • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 108 | 109 | 110 |
    111 | 112 |
    113 |
    114 | 119 | 120 |
    121 | 122 |

    Is the current state valid (stored in the cache)?

    123 | 124 |
    125 | 126 |
    is_valid(flow, state = "current")
    127 | 128 |

    Arguments

    129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 140 | 141 |
    flow

    A flow object, e.g. as returned by flow_fn.

    state

    A flow state. It can be either a valid state 138 | index (integer) or a valid state: "current", "all", 139 | in_hash or out_hash (string).

    142 | 143 |

    Value

    144 | 145 |

    A logical value, whether the value can be obtained without 146 | triggering computation.

    147 | 148 | 149 |

    Examples

    150 |
    fn <- function(x, y) { x + y + 9 } 151 | flowed_fn <- flow_fn(2, 3, fn = fn)
    #> New cache: fn=fn / fn_id=1 / fn_key=f802f2343ab49835
    is_valid_flow <- is_valid(flowed_fn)
    152 |
    153 | 164 |
    165 | 166 |
    167 | 170 | 171 |
    172 |

    Site built with pkgdown.

    173 |
    174 | 175 |
    176 |
    177 | 178 | 179 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /docs/reference/to_ns.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Assigns a value to a variable in a name space. — to_ns • rflow 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 47 | 48 | 49 | 50 | 51 | 52 |
    53 |
    54 | 108 | 109 | 110 |
    111 | 112 |
    113 |
    114 | 119 | 120 |
    121 | 122 |

    Assigns a value to a variable in a name space.

    123 | 124 |
    125 | 126 |
    to_ns(x, var_name, ns)
    127 | 128 |

    Arguments

    129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 143 | 144 |
    x

    Value to assign.

    var_name

    The name (as string) of the variable.

    ns

    The name space, either an environment or a 142 | Shiny::reactiveValues object.

    145 | 146 |

    Value

    147 | 148 |

    The initial value, x

    149 | 150 | 151 |
    152 | 161 |
    162 | 163 | 173 |
    174 | 175 | 176 | 177 | 178 | 179 | 180 | -------------------------------------------------------------------------------- /man/cache_file.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/cache-functions.R 3 | \name{cache_file} 4 | \alias{cache_file} 5 | \title{Get the file only cache engine.} 6 | \usage{ 7 | cache_file(cache_dir) 8 | } 9 | \arguments{ 10 | \item{cache_dir}{Directory where to store the cache files.} 11 | } 12 | \value{ 13 | A cache object that inherits from \code{R6Cache}. 14 | } 15 | \description{ 16 | All cache data will be stored only on the local disk. 17 | } 18 | \examples{ 19 | \dontrun{ 20 | cache_in_file <- cache_file("path_to_cache_diretory") 21 | } 22 | 23 | } 24 | \seealso{ 25 | Other cache functions: \code{\link{cache_memory_file}}, 26 | \code{\link{cache_memory}}, \code{\link{default_cache}} 27 | } 28 | \concept{cache functions} 29 | -------------------------------------------------------------------------------- /man/cache_memory.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/cache-functions.R 3 | \name{cache_memory} 4 | \alias{cache_memory} 5 | \title{Get the memory only cache engine.} 6 | \usage{ 7 | cache_memory() 8 | } 9 | \value{ 10 | A cache object that inherits from \code{R6Cache}. 11 | } 12 | \description{ 13 | All cache data will be stored only in memory. 14 | } 15 | \examples{ 16 | cache_in_memory <- cache_memory() 17 | 18 | } 19 | \seealso{ 20 | Other cache functions: \code{\link{cache_file}}, 21 | \code{\link{cache_memory_file}}, 22 | \code{\link{default_cache}} 23 | } 24 | \concept{cache functions} 25 | -------------------------------------------------------------------------------- /man/cache_memory_file.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/cache-functions.R 3 | \name{cache_memory_file} 4 | \alias{cache_memory_file} 5 | \title{Get the memory-file cache engine.} 6 | \usage{ 7 | cache_memory_file(cache_dir) 8 | } 9 | \arguments{ 10 | \item{cache_dir}{Directory where to store the cache files.} 11 | } 12 | \value{ 13 | A cache object that inherits from \code{R6Cache}. 14 | } 15 | \description{ 16 | The cache data will be stored both in memory (for quick access) and on 17 | disk (for persistence). 18 | } 19 | \examples{ 20 | \dontrun{ 21 | cache_fmem <- cache_memory_file("path_to_cache_diretory") 22 | } 23 | 24 | 25 | } 26 | \seealso{ 27 | Other cache functions: \code{\link{cache_file}}, 28 | \code{\link{cache_memory}}, \code{\link{default_cache}} 29 | } 30 | \concept{cache functions} 31 | -------------------------------------------------------------------------------- /man/collect.R6Flow.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{collect} 4 | \alias{collect} 5 | \alias{collect.R6Flow} 6 | \alias{collect.Element} 7 | \title{Get the data from an \code{R6Flow} or an \code{Element} object} 8 | \usage{ 9 | \method{collect}{R6Flow}(x, ...) 10 | 11 | \method{collect}{Element}(x, ...) 12 | } 13 | \arguments{ 14 | \item{x}{A flow object, e.g. as returned by \code{\link{flow_fn}}.} 15 | 16 | \item{...}{Name of the element of the output data to be selected. 17 | If present, it must be named \code{name}, otherwise the first 18 | item of the \code{...} list will be used. 19 | The default is \code{name = NULL}, which returns all the data. 20 | Ignored if \code{x} is an \code{Element} object.} 21 | } 22 | \value{ 23 | Data associated with the output of the function. 24 | } 25 | \description{ 26 | Get the data from an \code{R6Flow} or an \code{Element} object 27 | } 28 | \examples{ 29 | fn <- function(x, y) { x + y + 4 } 30 | flowed_fn <- flow_fn(2, 3, fn = fn) 31 | flow_result <- flowed_fn \%>\% collect() 32 | 33 | fn <- function(x, y) { list(x = x, y = y, z = 5) } 34 | flowed_fn <- flow_fn(2, 3, fn = fn) 35 | flow_result <- flowed_fn \%>\% 36 | collect() 37 | flow_element <- element(flowed_fn, "x") 38 | collected_element_value <- collect(flow_element) 39 | 40 | } 41 | -------------------------------------------------------------------------------- /man/compute.R6Flow.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{compute} 4 | \alias{compute} 5 | \alias{compute.R6Flow} 6 | \alias{compute.Element} 7 | \title{Trigger computation for an \code{R6Flow} or an \code{Element} object} 8 | \usage{ 9 | \method{compute}{R6Flow}(x, ...) 10 | 11 | \method{compute}{Element}(x, ...) 12 | } 13 | \arguments{ 14 | \item{x}{A flow object, e.g. as returned by \code{\link{flow_fn}}.} 15 | 16 | \item{...}{Any other arguments will be ignored.} 17 | } 18 | \value{ 19 | Logical, whether the result is available to be collected. 20 | } 21 | \description{ 22 | Trigger computation for an \code{R6Flow} or an \code{Element} object 23 | } 24 | \details{ 25 | Unlike \code{collect}, it does not trigger an error if it fails 26 | to compute and it does not return the actual result of the computation. 27 | } 28 | -------------------------------------------------------------------------------- /man/default_cache.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/cache-functions.R 3 | \name{default_cache} 4 | \alias{default_cache} 5 | \title{Get the default cache engine.} 6 | \usage{ 7 | default_cache() 8 | } 9 | \value{ 10 | A cache object that inherits from \code{R6Cache}. 11 | } 12 | \description{ 13 | Currently, the default cache engine is \code{cache_memory()}, but this 14 | might change in the future. 15 | } 16 | \examples{ 17 | \dontrun{ 18 | current_cache <- default_cache() 19 | } 20 | 21 | } 22 | \seealso{ 23 | Other cache functions: \code{\link{cache_file}}, 24 | \code{\link{cache_memory_file}}, 25 | \code{\link{cache_memory}} 26 | } 27 | \concept{cache functions} 28 | -------------------------------------------------------------------------------- /man/default_eddy_env.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/eddy-functions.R 3 | \name{default_eddy_env} 4 | \alias{default_eddy_env} 5 | \title{Get the default binding environment that keeps the eddies} 6 | \usage{ 7 | default_eddy_env() 8 | } 9 | \value{ 10 | An environment. 11 | } 12 | \description{ 13 | Get the default binding environment that keeps the eddies 14 | } 15 | -------------------------------------------------------------------------------- /man/delete_eddy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/eddy-functions.R 3 | \name{delete_eddy} 4 | \alias{delete_eddy} 5 | \title{Delete eddy and ALL its data from ALL cache layers (memory and/or disk)} 6 | \usage{ 7 | delete_eddy(eddy_name, eddy_env = default_eddy_env()) 8 | } 9 | \arguments{ 10 | \item{eddy_name}{Unique name of the \code{eddy} to be deleted.} 11 | 12 | \item{eddy_env}{The environment in which the eddy is put (bound).} 13 | } 14 | \value{ 15 | Nothing (\code{NULL}). 16 | } 17 | \description{ 18 | Delete eddy and ALL its data from ALL cache layers (memory and/or disk) 19 | } 20 | \examples{ 21 | new_eddy("eddy_new") 22 | # processing steps ... 23 | delete_eddy("eddy_new") 24 | 25 | } 26 | \seealso{ 27 | Other eddy functions: \code{\link{get_current_eddy}}, 28 | \code{\link{get_eddy}}, \code{\link{new_eddy}}, 29 | \code{\link{set_current_eddy}}, \code{\link{use_eddy}} 30 | } 31 | \concept{eddy functions} 32 | -------------------------------------------------------------------------------- /man/element.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{element} 4 | \alias{element} 5 | \alias{[.R6Flow} 6 | \title{Extract an element from an \code{R6Flow} object} 7 | \usage{ 8 | element(flow, name = NULL) 9 | 10 | \method{[}{R6Flow}(flow, name) 11 | } 12 | \arguments{ 13 | \item{flow}{A flow object, e.g. as returned by \code{\link{flow_fn}}.} 14 | 15 | \item{name}{Element of the output data to be selected. The default is 16 | \code{name = NULL}, which returns the element version of the \code{R6Flow} 17 | input object.} 18 | } 19 | \value{ 20 | An object with class \code{Element}. 21 | } 22 | \description{ 23 | Extract an element from an \code{R6Flow} object 24 | } 25 | \examples{ 26 | fn <- function(x, y) { list(x = x, y = y, Z = 6) } 27 | flowed_fn <- flow_fn(2, 3, fn = fn) 28 | flow_element <- element(flowed_fn, "x") 29 | 30 | fn <- function(x, y) { list(x = x, y = y, Z = 7) } 31 | flowed_fn <- flow_fn(2, 3, fn = fn) 32 | element_name <- flowed_fn["x"]$elem_name 33 | 34 | } 35 | -------------------------------------------------------------------------------- /man/flow_call.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{flow_call} 4 | \alias{flow_call} 5 | \title{Implicit cache of a function and of the given call} 6 | \usage{ 7 | flow_call(fn_call, fn_id = NULL, flow_options = get_flow_options()) 8 | } 9 | \arguments{ 10 | \item{fn_call}{Function call to be processed.} 11 | 12 | \item{fn_id}{Character or Integer. Optional id to uniquely identify 13 | the function. By default, rflow functions reuse the \code{cache} if the 14 | same function is given. The \code{fn_id} allows the user to suppress 15 | console messages and to explicitly indicate whether to reuse the old 16 | \code{cache} or create a new one.} 17 | 18 | \item{flow_options}{List of options created using \code{get_flow_options}.} 19 | } 20 | \value{ 21 | The flow object. 22 | } 23 | \description{ 24 | Implicit cache of a function and of the given call 25 | } 26 | \examples{ 27 | fn <- function(x, y) { x + y + 3 } 28 | call_flow <- flow_call(fn(x = 1, y = 2)) 29 | collected_result <- call_flow \%>\% collect() 30 | 31 | } 32 | -------------------------------------------------------------------------------- /man/flow_dfg.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-dfg.R 3 | \name{flow_dfg} 4 | \alias{flow_dfg} 5 | \title{Group-wise caching of operations on data frame} 6 | \usage{ 7 | flow_dfg(..., fn = NULL, fn_id = NULL, group_by = NULL, 8 | flow_options = get_flow_options()) 9 | } 10 | \arguments{ 11 | \item{...}{Named arguments to pass to \code{fn}. The first argument must be 12 | a \code{data.frame} or \code{tibble}. Row names are not supported. 13 | If no \code{group_by} values are provided, the data frame must be grouped.} 14 | 15 | \item{fn}{The function to apply to the data frame. It must accept a data 16 | frame as the first argument. \code{fn} may also apply \code{group_by} 17 | operations if the data frame given as input is not already grouped.} 18 | 19 | \item{fn_id}{Optional id to uniquely identify the function. By default, 20 | rflow functions reuse the cache if the same function is given. The id 21 | allows the user to suppress console messages and to explicitly 22 | indicate whether to reuse the old cache or create a new one.} 23 | 24 | \item{group_by}{A character vector of column names. If provided, groups 25 | already present will be ignored.} 26 | 27 | \item{flow_options}{List of options created using \code{get_flow_options}.} 28 | } 29 | \value{ 30 | The flow object. 31 | } 32 | \description{ 33 | Group-wise caching of operations on data frame 34 | } 35 | \details{ 36 | Function \code{fn} will receive only the rows and groups changed; 37 | it may drop some of the rows, but will not add any new rows. 38 | The function \code{fn} may return fewer or more columns or modify 39 | existing columns as long it always returns a consistent schema 40 | (i.e., the same column data types and names) for all calls. 41 | The data frame \code{df} passed to \code{fn} will include two 42 | additional columns: \code{..row_hash..} and \code{..group_hash..} that 43 | must be returned as is in order to identify changes. 44 | 45 | Arguments \code{fn}, \code{fn_id} and \code{flow_options}, when provided, 46 | must be named. Argument \code{fn} must be always provided. 47 | } 48 | \examples{ 49 | dfg_fn <- function(df) { 50 | df <- df \%>\% 51 | dplyr::mutate(Sepal.Length = Sepal.Length * 2) 52 | } 53 | 54 | dfg_fn2 <- function(df) { 55 | df <- df \%>\% 56 | dplyr::mutate(Petal.Length = Petal.Length * 3) 57 | } 58 | 59 | iris <- iris \%>\% 60 | dplyr::group_by(Species) 61 | dfg_flow <- flow_dfg(iris, fn = dfg_fn) 62 | collected_dfg <- dfg_flow \%>\% collect() 63 | 64 | # when a change in group is made, the flow object updates its state 65 | iris[1, "Species"] <- "virginica" 66 | dfg_flow <- flow_dfg(iris, fn = dfg_fn) 67 | collected_dfg <- dfg_flow \%>\% collect() 68 | 69 | # the flow element can also become input for another flow_dfg function 70 | # in order to allow multiple, chained computations 71 | collected_dfg2 <- dfg_flow \%>\% 72 | flow_dfg(fn = dfg_fn2, group_by = "Species") \%>\% 73 | collect() 74 | } 75 | -------------------------------------------------------------------------------- /man/flow_dfr.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-dfr.R 3 | \name{flow_dfr} 4 | \alias{flow_dfr} 5 | \title{Row-wise caching of operations on data frame} 6 | \usage{ 7 | flow_dfr(..., fn = NULL, fn_id = NULL, 8 | flow_options = get_flow_options()) 9 | } 10 | \arguments{ 11 | \item{...}{Named arguments to pass to \code{fn}. The first argument must be 12 | a \code{data.frame} or \code{tibble}. Row names are not supported.} 13 | 14 | \item{fn}{The function to apply to the data frame. It must accept a data 15 | frame as the first argument.} 16 | 17 | \item{fn_id}{Optional id to uniquely identify the function. By default, 18 | rflow functions reuse the \code{cache} if the same function is given. The id 19 | allows the user to suppress console messages and to explicitly 20 | indicate whether to reuse the old cache or create a new one.} 21 | 22 | \item{flow_options}{List of options created using \code{get_flow_options}.} 23 | } 24 | \value{ 25 | The flow object. 26 | } 27 | \description{ 28 | Row-wise caching of operations on data frame 29 | } 30 | \details{ 31 | Function \code{fn} operates on a data frame received as argument. 32 | \code{fn} will receive only the rows changed; 33 | it may drop some of the rows, but will not add any new rows. 34 | The function \code{fn} may return fewer or more columns or modify 35 | existing columns as long it always returns a consistent schema 36 | (i.e., the same column data types and names) for all calls. 37 | The data frame \code{df} passed to \code{fn} will include one 38 | additional column \code{..row_hash..} that must be returned as is in 39 | order to identify changes. 40 | 41 | Arguments \code{fn}, \code{fn_id} and \code{flow_options}, when provided, 42 | must be named. Argument \code{fn} must be always provided. 43 | } 44 | \examples{ 45 | df_fn <- function(df, i = NULL) { 46 | if (is.null(i)) { 47 | dfi <- df 48 | dfi$rm <- rowMeans(dfi[1:10]) 49 | } else { 50 | dfi <- df[i, , drop = FALSE] 51 | } 52 | dfi 53 | } 54 | 55 | # the flow element can also become input for another flow_df function 56 | # in order to allow multiple, chained computations 57 | dfr_flow <- flow_dfr(mtcars, 1, fn = df_fn) 58 | collected_dfr <- dfr_flow \%>\% 59 | collect() 60 | 61 | } 62 | -------------------------------------------------------------------------------- /man/flow_file_source.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/source-file.R 3 | \name{flow_file_source} 4 | \alias{flow_file_source} 5 | \title{Creates a flow object that watches one or more files} 6 | \usage{ 7 | flow_file_source(file_path, flow_options = get_flow_options()) 8 | } 9 | \arguments{ 10 | \item{file_path}{A (named) vector of file paths to be watched or a 11 | \code{fs_path} object.} 12 | 13 | \item{flow_options}{List of options created using \code{get_flow_options}. 14 | All options except \code{eddy} are ignored.} 15 | } 16 | \value{ 17 | The flow object. 18 | } 19 | \description{ 20 | Creates a flow object that watches one or more files 21 | } 22 | \details{ 23 | This flow object does not throw an error if the \code{file_path} is missing, 24 | but it changes its state. Hence, it can be used to trigger a downstream 25 | flow object if the file is now present, changed or missing. 26 | } 27 | \examples{ 28 | # write for the first time content in file and create flow 29 | file_temp <- tempfile(pattern = "example_source") 30 | write.csv(head(mtcars), file_temp, row.names = FALSE) 31 | rflow_source <- flow_file_source(file_temp) 32 | 33 | # write other content in the same file 34 | # now the flow object will update its state 35 | write.csv(tail(mtcars), file_temp, row.names = FALSE) 36 | rflow_source <- flow_file_source(file_temp) 37 | unlink(file_temp) 38 | 39 | } 40 | -------------------------------------------------------------------------------- /man/flow_fn.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{flow_fn} 4 | \alias{flow_fn} 5 | \title{Implicit cache of a function and of the given call} 6 | \usage{ 7 | flow_fn(..., fn = NULL, fn_id = NULL, 8 | flow_options = get_flow_options()) 9 | } 10 | \arguments{ 11 | \item{...}{Named arguments to pass to \code{fn}.} 12 | 13 | \item{fn}{The function to apply to the data frame. It must accept a data 14 | frame as the first argument and a numeric index as the second argument.} 15 | 16 | \item{fn_id}{Optional id to uniquely identify the function. By default, 17 | rflow functions reuse the \code{cache} if the same function is given. The id 18 | allows the user to suppress console messages and to explicitly 19 | indicate whether to reuse the old \code{cache} or create a new one.} 20 | 21 | \item{flow_options}{List of options created using \code{get_flow_options}.} 22 | } 23 | \value{ 24 | The flow object. 25 | } 26 | \description{ 27 | Implicit cache of a function and of the given call 28 | } 29 | \details{ 30 | Arguments \code{fn}, \code{fn_id} and \code{flow_options}, when provided, 31 | must be named. Argument \code{fn} must be always provided. 32 | } 33 | \examples{ 34 | fn <- function(x, y) { x + y + 2 } 35 | flowed_fn <- flow_fn(2, 3, fn = fn) 36 | collected_result <- flowed_fn \%>\% 37 | collect() 38 | # usage with rflow pipes 39 | fn2 <- function(x, y) { x * y } 40 | collected_pipe_result <- flowed_fn \%>\% 41 | flow_fn(2, fn = fn2) \%>\% 42 | collect() 43 | 44 | } 45 | -------------------------------------------------------------------------------- /man/flow_ns_sink.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/sink-ns.R 3 | \name{flow_ns_sink} 4 | \alias{flow_ns_sink} 5 | \title{Write a value to a namespace only if the value has changed.} 6 | \usage{ 7 | flow_ns_sink(x, var_name, ns, flow_options = get_flow_options()) 8 | } 9 | \arguments{ 10 | \item{x}{Value to assign.} 11 | 12 | \item{var_name}{The name (as string) of the variable.} 13 | 14 | \item{ns}{The name space, either an \code{environment} or a 15 | \code{Shiny::reactiveValues} object.} 16 | 17 | \item{flow_options}{List of options created using \code{get_flow_options}. 18 | All options except \code{excluded_arg} and \code{eddy} are ignored.} 19 | } 20 | \value{ 21 | The flow object 22 | } 23 | \description{ 24 | Write a value to a namespace only if the value has changed. 25 | } 26 | \examples{ 27 | fn <- function(x, y) { x + y } 28 | flowed_fn <- flow_fn(1, 2, fn = fn) 29 | sunk_flow <- flow_ns_sink(flowed_fn, "test_sink_flow", new.env()) 30 | 31 | } 32 | -------------------------------------------------------------------------------- /man/flow_options.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/eddy-functions.R 3 | \name{flow_options} 4 | \alias{flow_options} 5 | \alias{default_flow_options} 6 | \alias{set_flow_options} 7 | \alias{get_flow_options} 8 | \title{flow options used to initialize or update an eddy} 9 | \usage{ 10 | default_flow_options(excluded_arg = character(), 11 | split_bare_list = TRUE, split_dataframe = FALSE, split_fn = NULL) 12 | 13 | set_flow_options(excluded_arg = NULL, split_bare_list = NULL, 14 | split_dataframe = NULL, split_fn = NULL, eddy = get_current_eddy()) 15 | 16 | get_flow_options(excluded_arg = NULL, eval_arg_fn = NULL, 17 | split_bare_list = NULL, split_dataframe = NULL, split_fn = NULL, 18 | eddy = get_current_eddy()) 19 | } 20 | \arguments{ 21 | \item{excluded_arg}{A vector of argument names to be excluded when computing 22 | the input hash. Best used to exclude certain arguments that depend on 23 | the running state, e.g. a Shiny session, a parallel cluster, etc. 24 | Excluded arguments must have a default value to permit lazy computations. 25 | The default is not to exclude any arguments from the input hash.} 26 | 27 | \item{split_bare_list}{If the function output is a bare list 28 | (\code{\link[rlang:bare-type-predicates]{rlang::is_bare_list}}), determines 29 | whether to calculate the hash of each list element and create 30 | corresponding flow elements.} 31 | 32 | \item{split_dataframe}{If the function output is a data.frame or tibble, 33 | determines whether to calculate the hash of each column and create 34 | corresponding flow elements.} 35 | 36 | \item{split_fn}{Custom function to generate a list of elements from the 37 | output of the flow-ed function. Useful only if the output is not a list 38 | but a flow elements are still desired. Consider returning a list 39 | as output before using this option. If an \code{split_fn} 40 | is provided, \code{split_bare_list} and \code{split_dataframe} will be 41 | ignored.} 42 | 43 | \item{eddy}{Eddy to apply / retrieve options to / from.} 44 | 45 | \item{eval_arg_fn}{Custom function to parse the input arguments and create 46 | a list of evaluated arguments to be hashed. This function should have the 47 | exact same arguments as the original function. Try to use 48 | \code{excluded_arg} or flow source before creating a custom function. 49 | Because each custom function is flow specific, it is not possible to set 50 | this option at the eddy level using \code{set_flow_options}.} 51 | } 52 | \value{ 53 | For \code{default_flow_options}, a list of options. 54 | 55 | For \code{set_flow_options}, \code{NULL}. 56 | 57 | For \code{get_flow_options}, a list of options 58 | including the eddy. 59 | } 60 | \description{ 61 | flow options used to initialize or update an eddy 62 | 63 | default flow options used to initialize or update an eddy. 64 | } 65 | \details{ 66 | If used in \code{set_flow_options}, these options will be stored in 67 | \code{eddy} and retrieved by each flow subsequently executed. E.g. if not 68 | careful, it is possible to force all following flows to use a custom 69 | \code{split_fn} function, which is not recommended. 70 | 71 | Sets a list of default options. 72 | 73 | \code{set_flow_options} does not overwrite the current options when the 74 | argument is \code{NULL}. 75 | } 76 | -------------------------------------------------------------------------------- /man/forget.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{forget} 4 | \alias{forget} 5 | \title{Forgets the computation for the current state} 6 | \usage{ 7 | forget(flow, state = "current") 8 | } 9 | \arguments{ 10 | \item{flow}{A flow object, e.g. as returned by \code{\link{flow_fn}}.} 11 | 12 | \item{state}{A flow state. It can be either a valid state 13 | \code{index} (integer) or a valid state: \code{"current"}, \code{"all"}, 14 | \code{in_hash} or \code{out_hash} (string).} 15 | } 16 | \value{ 17 | A logical value, whether the deletion was successful. 18 | } 19 | \description{ 20 | Forgets the computation for the current state 21 | } 22 | \examples{ 23 | fn <- function(x, y) { x + y + 10 } 24 | flowed_fn <- flow_fn(2, 3, fn = fn) 25 | forget(flowed_fn) 26 | 27 | } 28 | -------------------------------------------------------------------------------- /man/get_current_eddy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/eddy-functions.R 3 | \name{get_current_eddy} 4 | \alias{get_current_eddy} 5 | \title{Get the current eddy for a given (or default) environment} 6 | \usage{ 7 | get_current_eddy(eddy_env = default_eddy_env()) 8 | } 9 | \arguments{ 10 | \item{eddy_env}{The environment in which the eddy is put (bound).} 11 | } 12 | \value{ 13 | An eddy object to be used for storing flows. 14 | } 15 | \description{ 16 | If the current eddy was not previously set with \code{set_current_eddy}, 17 | it creates a new eddy that uses \code{default_cache()}. 18 | } 19 | \examples{ 20 | get_current_eddy() 21 | 22 | } 23 | \seealso{ 24 | Other eddy functions: \code{\link{delete_eddy}}, 25 | \code{\link{get_eddy}}, \code{\link{new_eddy}}, 26 | \code{\link{set_current_eddy}}, \code{\link{use_eddy}} 27 | } 28 | \concept{eddy functions} 29 | -------------------------------------------------------------------------------- /man/get_eddy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/eddy-functions.R 3 | \name{get_eddy} 4 | \alias{get_eddy} 5 | \title{Retrieves an eddy} 6 | \usage{ 7 | get_eddy(eddy_name, eddy_env = default_eddy_env()) 8 | } 9 | \arguments{ 10 | \item{eddy_name}{Unique name for the eddy to allow retrieving later.} 11 | 12 | \item{eddy_env}{The environment in which the \code{eddy}is put (bound).} 13 | } 14 | \value{ 15 | An eddy object to be used for storing flows. 16 | } 17 | \description{ 18 | Retrieves an eddy 19 | } 20 | \examples{ 21 | new_eddy("eddy_name") 22 | get_eddy("eddy_name") 23 | 24 | } 25 | \seealso{ 26 | Other eddy functions: \code{\link{delete_eddy}}, 27 | \code{\link{get_current_eddy}}, \code{\link{new_eddy}}, 28 | \code{\link{set_current_eddy}}, \code{\link{use_eddy}} 29 | } 30 | \concept{eddy functions} 31 | -------------------------------------------------------------------------------- /man/imports.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/package.R 3 | \name{\%>\%} 4 | \alias{\%>\%} 5 | \alias{\%||\%} 6 | \title{Imports from other packages} 7 | \description{ 8 | Imports from other packages 9 | } 10 | \keyword{internal} 11 | -------------------------------------------------------------------------------- /man/is_current.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{is_current} 4 | \alias{is_current} 5 | \title{Checks if the flow has a"current" state} 6 | \usage{ 7 | is_current(flow) 8 | } 9 | \arguments{ 10 | \item{flow}{A flow object, e.g. as returned by \code{\link{flow_fn}}.} 11 | } 12 | \value{ 13 | A logical value, whether the current state is valid. 14 | } 15 | \description{ 16 | If there is no current state, e.g. right after \code{\link{make_flow_fn}}, 17 | the flow is "not flowing", it is preventing downstream flows 18 | from being computed. 19 | } 20 | \examples{ 21 | fn <- function(x, y) { x + y + 8 } 22 | flowed_fn <- flow_fn(1, 2, fn = fn) 23 | is_current_flow <- is_current(flowed_fn) 24 | 25 | } 26 | -------------------------------------------------------------------------------- /man/is_flow.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{is_flow} 4 | \alias{is_flow} 5 | \title{Checks if the object is flow object or a flow element} 6 | \usage{ 7 | is_flow(x) 8 | } 9 | \arguments{ 10 | \item{x}{An object.} 11 | } 12 | \value{ 13 | A logical value, whether \code{x} is a flow object. 14 | } 15 | \description{ 16 | Checks if the object is flow object or a flow element 17 | } 18 | \examples{ 19 | fn <- function(x, y) { x + y + 11 } 20 | flow_function <- flow_fn(1, 2, fn = fn) 21 | is_input_flow <- is_flow(flow_function) 22 | 23 | } 24 | -------------------------------------------------------------------------------- /man/is_flow_fn.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{is_flow_fn} 4 | \alias{is_flow_fn} 5 | \title{Checks if the function is flow function (as returned by \code{make_flow_fn})} 6 | \usage{ 7 | is_flow_fn(fn) 8 | } 9 | \arguments{ 10 | \item{fn}{A function.} 11 | } 12 | \value{ 13 | A logical value, whether \code{fn} is a flow function. 14 | } 15 | \description{ 16 | Checks if the function is flow function (as returned by \code{make_flow_fn}) 17 | } 18 | \examples{ 19 | fn <- function(x, y) { x + y + 12 } 20 | flowed_function <- flow_fn(2, 3, fn = fn) 21 | is_flow_function <- is_flow_fn(fn = flowed_function) 22 | 23 | } 24 | -------------------------------------------------------------------------------- /man/is_not_flow_fn.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{is_not_flow_fn} 4 | \alias{is_not_flow_fn} 5 | \title{Check if the function is NOT a flow function 6 | (as returned by \code{make_flow_fn}).} 7 | \usage{ 8 | is_not_flow_fn(fn) 9 | } 10 | \arguments{ 11 | \item{fn}{A function.} 12 | } 13 | \value{ 14 | A logical value, whether \code{fn} is a not flow function. 15 | } 16 | \description{ 17 | Check if the function is NOT a flow function 18 | (as returned by \code{make_flow_fn}). 19 | } 20 | -------------------------------------------------------------------------------- /man/is_valid.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{is_valid} 4 | \alias{is_valid} 5 | \title{Checks if the current state is valid (stored in the cache)} 6 | \usage{ 7 | is_valid(flow, state = "current") 8 | } 9 | \arguments{ 10 | \item{flow}{A flow object, e.g. as returned by \code{\link{flow_fn}}.} 11 | 12 | \item{state}{A flow state. It can be either a valid state 13 | \code{index} (integer) or a valid state: \code{"current"}, \code{"all"}, 14 | \code{in_hash} or \code{out_hash} (string).} 15 | } 16 | \value{ 17 | A logical value, whether the value can be obtained without 18 | triggering computation. 19 | } 20 | \description{ 21 | Checks if the current state is valid (stored in the cache) 22 | } 23 | \examples{ 24 | fn <- function(x, y) { x + y + 9 } 25 | flowed_fn <- flow_fn(2, 3, fn = fn) 26 | is_valid_flow <- is_valid(flowed_fn) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /man/make_flow_fn.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/flow-functions.R 3 | \name{make_flow_fn} 4 | \alias{make_flow_fn} 5 | \title{Explicit cache of a function} 6 | \usage{ 7 | make_flow_fn(fn, fn_id = NULL, flow_options = get_flow_options()) 8 | } 9 | \arguments{ 10 | \item{fn}{Function to be cached, ideally a pure function.} 11 | 12 | \item{fn_id}{Optional id to uniquely identify the function. By default, 13 | rflow functions reuse the \code{cache} if the same function is given. The id 14 | allows the user to suppress console messages and to explicitly 15 | indicate whether to reuse the old \code{cache} or create a new one.} 16 | 17 | \item{flow_options}{List of options created using \code{get_flow_options}.} 18 | } 19 | \value{ 20 | The cached version of the function. 21 | } 22 | \description{ 23 | Explicit cache of a function 24 | } 25 | \details{ 26 | In order to use the functionality of an \code{R6Flow} object, the output of 27 | \code{make_flow_fn} has to be collected first. 28 | } 29 | \examples{ 30 | fn <- function(x, y) { x + y + 1 } 31 | make_flow_function <- make_flow_fn(fn) 32 | rflow_function <- make_flow_function(2, 3) 33 | flow_result <- rflow_function \%>\% collect() 34 | 35 | # usage with rflow pipes 36 | fn2 <- function(x, y) { x * y } 37 | make_flow_function2 <- make_flow_fn(fn2) 38 | collected_pipe_flow <- make_flow_function(1, 2) \%>\% 39 | make_flow_function2(2) \%>\% 40 | collect() 41 | 42 | } 43 | -------------------------------------------------------------------------------- /man/new_eddy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/eddy-functions.R 3 | \name{new_eddy} 4 | \alias{new_eddy} 5 | \title{Create a new eddy. Does not affect the current eddy} 6 | \usage{ 7 | new_eddy(eddy_name, cache = default_cache(), 8 | flow_options = default_flow_options(), eddy_env = default_eddy_env()) 9 | } 10 | \arguments{ 11 | \item{eddy_name}{Unique name for the eddy to allow retrieving later.} 12 | 13 | \item{cache}{A cache object returned by one of the \code{cache} functions.} 14 | 15 | \item{flow_options}{Options to store for future flow invocations. They 16 | do not affect the \code{eddy}, they are only stored for ease of access.} 17 | 18 | \item{eddy_env}{The environment in which the \code{eddy} is put (bound).} 19 | } 20 | \value{ 21 | An eddy object to be used for storing flows. 22 | } 23 | \description{ 24 | Create a new eddy. Does not affect the current eddy 25 | } 26 | \examples{ 27 | new_eddy("new_eddy_name") 28 | 29 | } 30 | \seealso{ 31 | Other eddy functions: \code{\link{delete_eddy}}, 32 | \code{\link{get_current_eddy}}, \code{\link{get_eddy}}, 33 | \code{\link{set_current_eddy}}, \code{\link{use_eddy}} 34 | } 35 | \concept{eddy functions} 36 | -------------------------------------------------------------------------------- /man/set_current_eddy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/eddy-functions.R 3 | \name{set_current_eddy} 4 | \alias{set_current_eddy} 5 | \title{Set the current eddy to be used in future flow calls} 6 | \usage{ 7 | set_current_eddy(eddy_name, eddy_env = default_eddy_env()) 8 | } 9 | \arguments{ 10 | \item{eddy_name}{Unique name for the \code{eddy} to to be set as current.} 11 | 12 | \item{eddy_env}{The environment in which the eddy is put (bound).} 13 | } 14 | \value{ 15 | An eddy object to be used for storing flows. 16 | } 17 | \description{ 18 | Set the current eddy to be used in future flow calls 19 | } 20 | \examples{ 21 | new_eddy("eddy") 22 | set_current_eddy("eddy") 23 | 24 | } 25 | \seealso{ 26 | Other eddy functions: \code{\link{delete_eddy}}, 27 | \code{\link{get_current_eddy}}, \code{\link{get_eddy}}, 28 | \code{\link{new_eddy}}, \code{\link{use_eddy}} 29 | } 30 | \concept{eddy functions} 31 | -------------------------------------------------------------------------------- /man/to_ns.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/sink-ns.R 3 | \name{to_ns} 4 | \alias{to_ns} 5 | \title{Assigns a value to a variable in a name space} 6 | \usage{ 7 | to_ns(x, var_name, ns) 8 | } 9 | \arguments{ 10 | \item{x}{Value to assign.} 11 | 12 | \item{var_name}{The name (as string) of the variable.} 13 | 14 | \item{ns}{The name space, either an \code{environment} or a 15 | \code{Shiny::reactiveValues} object.} 16 | } 17 | \value{ 18 | The initial value, \code{x} 19 | } 20 | \description{ 21 | Assigns a value to a variable in a name space 22 | } 23 | -------------------------------------------------------------------------------- /man/use_eddy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/eddy-functions.R 3 | \name{use_eddy} 4 | \alias{use_eddy} 5 | \title{Convenience function that creates a new eddy or re-uses it if present} 6 | \usage{ 7 | use_eddy(eddy_name, cache = default_cache(), 8 | flow_options = default_flow_options(), reuse_if_present = TRUE, 9 | set_current = TRUE, eddy_env = default_eddy_env()) 10 | } 11 | \arguments{ 12 | \item{eddy_name}{Unique name for the eddy to allow retrieving later.} 13 | 14 | \item{cache}{A cache object returned by one of the \code{cache} functions.} 15 | 16 | \item{flow_options}{Options to store for future flow invocations. They 17 | do not affect the \code{eddy}, they are only stored for ease of access.} 18 | 19 | \item{reuse_if_present}{Logical, whether or not to reuse an eddy with 20 | the same name, if already present, even if the \code{flow_options} 21 | are different.} 22 | 23 | \item{set_current}{Logical, whether to make this eddy current.} 24 | 25 | \item{eddy_env}{The environment in which the eddy is put (bound).} 26 | } 27 | \value{ 28 | An eddy object to be used for storing flows. 29 | } 30 | \description{ 31 | Convenience function that creates a new eddy or re-uses it if present 32 | } 33 | \examples{ 34 | use_eddy("use_eddy") 35 | 36 | } 37 | \seealso{ 38 | Other eddy functions: \code{\link{delete_eddy}}, 39 | \code{\link{get_current_eddy}}, \code{\link{get_eddy}}, 40 | \code{\link{new_eddy}}, \code{\link{set_current_eddy}} 41 | } 42 | \concept{eddy functions} 43 | -------------------------------------------------------------------------------- /rflow.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: No 4 | SaveWorkspace: No 5 | AlwaysSaveHistory: Yes 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 4 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | 17 | BuildType: Package 18 | PackageUseDevtools: Yes 19 | PackageInstallArgs: --no-multiarch --with-keep.source 20 | PackageCheckArgs: --as-cran 21 | PackageRoxygenize: rd,collate,namespace,vignette 22 | 23 | QuitChildProcessesOnExit: Yes 24 | -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | library(rflow) 3 | 4 | test_check("rflow") 5 | -------------------------------------------------------------------------------- /tests/testthat/test-eddy-functions.R: -------------------------------------------------------------------------------- 1 | # Eddy functions tests ---- 2 | context("Eddy functions tests") 3 | 4 | if (digest::digest(Sys.info()[-c(2, 3)]) %in% c( 5 | "2e85e2a3018ecf3b2e5fc03bfb20fd39" 6 | )) { 7 | skip("cache-memory-file functions") 8 | } 9 | 10 | 11 | test_that("new_eddy() works", { 12 | eddy <- new_eddy("new_eddy_test") 13 | expect_equal(get_eddy("new_eddy_test"), eddy) 14 | delete_eddy("new_eddy_test") 15 | }) 16 | 17 | 18 | test_that("set_current_eddy() works", { 19 | eddy <- new_eddy("eddy_test") 20 | set_current_eddy("eddy_test") 21 | expect_equal(get_current_eddy(), eddy) 22 | delete_eddy("eddy_test") 23 | }) 24 | 25 | 26 | test_that("parse_flow_options() works", { 27 | 28 | eval_arg_fn <- function(x) {list(x)} 29 | eddy <- new_eddy(eddy_name = "test_eddy") 30 | options <- parse_flow_options( 31 | excluded_arg = "x", 32 | eval_arg_fn = eval_arg_fn, 33 | split_bare_list = TRUE, 34 | split_dataframe = FALSE, 35 | split_fn = NULL, 36 | eddy = eddy) 37 | 38 | expected_val <- list( 39 | excluded_arg = "x", 40 | eval_arg_fn = eval_arg_fn, 41 | split_bare_list = TRUE, 42 | split_dataframe = FALSE, 43 | split_fn = NULL) 44 | 45 | expect_equal(options, expected_val) 46 | delete_eddy("test_eddy") 47 | }) 48 | 49 | 50 | test_that("parse_flow_options() works with NULL args", { 51 | 52 | options <- parse_flow_options( 53 | excluded_arg = "x", 54 | eval_arg_fn = NULL, 55 | split_bare_list = TRUE, 56 | split_dataframe = FALSE, 57 | split_fn = NULL, 58 | eddy = NULL) 59 | 60 | expected_val <- list( 61 | excluded_arg = "x", 62 | eval_arg_fn = NULL, 63 | split_bare_list = TRUE, 64 | split_dataframe = FALSE, 65 | split_fn = NULL) 66 | 67 | expect_equal(options, expected_val) 68 | }) 69 | -------------------------------------------------------------------------------- /tests/testthat/test-flow-dfr.R: -------------------------------------------------------------------------------- 1 | # Tests for flow_dfr ----------------------------------------------------------- 2 | context("flow-dfr tests") 3 | 4 | if (digest::digest(Sys.info()[-c(2, 3)]) %in% c( 5 | "2e85e2a3018ecf3b2e5fc03bfb20fd39" 6 | )) { 7 | skip("cache-memory-file functions") 8 | } 9 | 10 | setup({ 11 | df <- tibble::as.tibble(mtcars) 12 | df_fn <- function(df, i = NULL) { 13 | if (is.null(i)) { 14 | dfi <- df 15 | dfi$rm <- rowMeans(dfi[1:10]) 16 | } else { 17 | dfi <- df[i, , drop = FALSE] 18 | } 19 | dfi 20 | } 21 | 22 | df_fn2 <- function(df) { 23 | df <- df %>% 24 | dplyr::mutate(hp = hp + 1) 25 | df 26 | } 27 | 28 | assign("df", df, envir = .GlobalEnv) 29 | assign("df_fn", df_fn, envir = .GlobalEnv) 30 | assign("df_fn2", df_fn2, envir = .GlobalEnv) 31 | }) 32 | 33 | 34 | test_that("flow_dfr() works", { 35 | get_current_eddy()$reset() 36 | dfr1 <- flow_dfr(head(df), fn = df_fn) 37 | 38 | expect_equal(dfr1$state_index, 1) 39 | expect_false(dfr1$is_valid) 40 | 41 | collected_dfr <- dfr1 %>% collect() 42 | expected_df <- head(df) 43 | expected_df$rm <- rowMeans(expected_df[1:10]) 44 | 45 | expect_true(dfr1$is_valid) 46 | expect_equal(collected_dfr, expected_df) 47 | }) 48 | 49 | 50 | test_that("flow_dfr() works with factor column", { 51 | get_current_eddy()$reset() 52 | test_df <- head(df) 53 | 54 | test_df$gear <- as.factor(test_df$gear) 55 | dfr1 <- flow_dfr(test_df, fn = df_fn2) 56 | 57 | expect_equal(dfr1$state_index, 1) 58 | expect_false(dfr1$is_valid) 59 | 60 | collected_dfr <- dfr1 %>% collect() 61 | expected_df <- head(df) %>% 62 | dplyr::mutate(hp = hp + 1, 63 | gear = as.factor(gear)) 64 | 65 | expect_true(dfr1$is_valid) 66 | expect_equal(collected_dfr, expected_df) 67 | expect_true(is.factor(collected_dfr$gear)) 68 | }) 69 | 70 | 71 | test_that("flow_dfr() works with second function argument", { 72 | get_current_eddy()$reset() 73 | 74 | df_test <- head(df) 75 | 76 | dfr1 <- flow_dfr(df_test, 1, fn = df_fn) 77 | 78 | expect_equal(dfr1$state_index, 1) 79 | expect_false(dfr1$is_valid) 80 | 81 | collected_dfr <- dfr1 %>% collect() 82 | expected_df <- df_test[1, , drop = FALSE] 83 | 84 | expect_true(dfr1$is_valid) 85 | expect_equal(collected_dfr, expected_df) 86 | }) 87 | 88 | 89 | test_that("flow_dfr() works with different states", { 90 | get_current_eddy()$reset() 91 | 92 | dfr1 <- flow_dfr(head(df), fn = df_fn) 93 | 94 | collected_dfr1 <- dfr1 %>% collect() 95 | expect_equal(dfr1$state_index, 1) 96 | expect_equal(nrow(dfr1$out_df), 6) 97 | 98 | expect_message(dfr1 <- flow_dfr(tail(df), fn = df_fn)) 99 | 100 | collected_dfr1 <- dfr1 %>% collect() 101 | expect_equal(dfr1$state_index, 2) 102 | expect_equal(nrow(dfr1$out_df), 12) 103 | expected_df <- tail(df) 104 | expected_df$rm <- rowMeans(expected_df[1:10]) 105 | 106 | expect_true(dfr1$is_valid) 107 | expect_equal(collected_dfr1, expected_df) 108 | }) 109 | 110 | 111 | test_that("flow_dfr() with same name, but different body", { 112 | get_current_eddy()$reset() 113 | 114 | dfr_test <- flow_dfr(head(df), fn = df_fn) 115 | expect_equal(dfr_test$state_index, 1) 116 | 117 | df_fn <- function(df, i = NULL) { 118 | if (is.null(i)) { 119 | dfi <- df 120 | } else { 121 | dfi <- df[i, , drop = FALSE] 122 | } 123 | 124 | dfi$mr <- rowSums(dfi[1:10]) 125 | dfi 126 | } 127 | assign("df_fn", df_fn, envir = .GlobalEnv) 128 | 129 | expect_message(dfr_test <- flow_dfr(head(df), fn = df_fn)) 130 | collected_dfr <- dfr_test %>% 131 | collect() 132 | 133 | expected_df <- head(df) 134 | expected_df$mr <- rowSums(expected_df[1:10]) 135 | 136 | expect_equal(dfr_test$state_index, 1) 137 | expect_equal(collected_dfr, expected_df) 138 | 139 | df_fn <- function(df, i = NULL) { 140 | if (is.null(i)) { 141 | dfi <- df 142 | } else { 143 | dfi <- df[i, , drop = FALSE] 144 | } 145 | dfi$rm <- rowMeans(dfi[1:10]) 146 | dfi 147 | } 148 | 149 | assign("df_fn", df_fn, envir = .GlobalEnv) 150 | }) 151 | 152 | 153 | test_that("flow_dfr() with same name, but different body with id", { 154 | get_current_eddy()$reset() 155 | 156 | dfr_test <- flow_dfr(head(df), fn = df_fn) 157 | expect_equal(dfr_test$state_index, 1) 158 | 159 | df_fn <- function(df, i = NULL) { 160 | if (is.null(i)) { 161 | dfi <- df 162 | } else { 163 | dfi <- df[i, , drop = FALSE] 164 | } 165 | 166 | dfi$mr <- rowSums(dfi[1:10]) 167 | dfi 168 | } 169 | assign("df_fn", df_fn, envir = .GlobalEnv) 170 | 171 | expect_silent( 172 | dfr_test <- flow_dfr(head(df), fn = df_fn, fn_id = "id1")) 173 | collected_dfr <- dfr_test %>% 174 | collect() 175 | 176 | expected_df <- head(df) 177 | expected_df$mr <- rowSums(expected_df[1:10]) 178 | 179 | expect_equal(dfr_test$state_index, 1) 180 | expect_equal(collected_dfr, expected_df) 181 | 182 | df_fn <- function(df, i = NULL) { 183 | if (is.null(i)) { 184 | dfi <- df 185 | } else { 186 | dfi <- df[i, , drop = FALSE] 187 | } 188 | dfi$rm <- rowMeans(dfi[1:10]) 189 | dfi 190 | } 191 | 192 | assign("df_fn", df_fn, envir = .GlobalEnv) 193 | }) 194 | 195 | 196 | test_that("flow_dfr() works with same body, different name", { 197 | get_current_eddy()$reset() 198 | 199 | df_fn2 <- function(df, i = NULL) { 200 | if (is.null(i)) { 201 | dfi <- df 202 | } else { 203 | dfi <- df[i, , drop = FALSE] 204 | } 205 | dfi$rm <- rowMeans(dfi[1:10]) 206 | dfi 207 | } 208 | assign("df_fn2", df_fn, envir = .GlobalEnv) 209 | 210 | dfr1 <- flow_dfr(head(df), fn = df_fn) 211 | 212 | expect_equal(dfr1$state_index, 1) 213 | expect_message(dfr2 <- flow_dfr(head(df), fn = df_fn2)) 214 | 215 | collected_dfr <- dfr2 %>% collect() 216 | expected_df <- head(df) 217 | expected_df$rm <- rowMeans(expected_df[1:10]) 218 | 219 | expect_equal(collected_dfr, expected_df) 220 | expect_equal(dfr2$state_index, 1) 221 | }) 222 | 223 | 224 | test_that("flow_dfr() stops with empty dataframe", { 225 | get_current_eddy()$reset() 226 | 227 | df <- data.frame() 228 | expect_error(dfr <- flow_dfr(df, fn = identity)) 229 | }) 230 | 231 | 232 | test_that("flow_dfr() works with 0 rows dataframe", { 233 | get_current_eddy()$reset() 234 | 235 | df <- data.frame(col1 = character(), col2 = character()) 236 | dfr <- flow_dfr(df, fn = identity) 237 | expect_equal(collect(dfr), tibble::as_tibble(df)) 238 | }) 239 | 240 | 241 | test_that("flow_dfr() stops with non df input", { 242 | get_current_eddy()$reset() 243 | 244 | expect_error(dfr <- flow_dfr(NULL, fn = identity)) 245 | expect_error(dfr <- flow_dfr(list(), fn = identity)) 246 | expect_error(dfr <- flow_dfr(NA, fn = identity)) 247 | expect_error(dfr <- flow_dfr(character(), fn = identity)) 248 | expect_error(dfr <- flow_dfr(1, fn = identity)) 249 | expect_error(dfr <- flow_dfr(TRUE, fn = identity)) 250 | }) 251 | 252 | 253 | teardown({ 254 | get_current_eddy()$reset() 255 | 256 | base::rm(list = "df", envir = .GlobalEnv) 257 | base::rm(list = "df_fn", envir = .GlobalEnv) 258 | base::rm(list = "df_fn2", envir = .GlobalEnv) 259 | }) 260 | -------------------------------------------------------------------------------- /tests/testthat/test-integrated.R: -------------------------------------------------------------------------------- 1 | # Integrated tests for cacheing ------------------------------------------------ 2 | 3 | context("Integrated tests") 4 | 5 | 6 | setup({ 7 | cache_dir <- tempfile(pattern = "cache-dir-") 8 | cache_fmem_test <- cache_memory_file(cache_dir) 9 | test_eddy <- use_eddy("test_eddy", cache = cache_fmem_test) 10 | 11 | df <- tibble::as.tibble(iris) 12 | df <- df[c(1:7, 71:73, 101:110), ] 13 | file_path <- as.character(fs::path(tempfile(pattern = "test-rflow-"))) 14 | 15 | df_fn <- function(df, i = NULL) { 16 | if (is.null(i)) { 17 | dfi <- df 18 | } else { 19 | dfi <- df[i, , drop = FALSE] 20 | } 21 | dfi %>% 22 | dplyr::group_by(Species) %>% 23 | dplyr::mutate(rm = mean(Sepal.Length)) 24 | } 25 | 26 | assign("cache_fmem_test", cache_fmem_test, envir = .GlobalEnv) 27 | assign("cache_dir", cache_dir, envir = .GlobalEnv) 28 | assign("file_path", file_path, envir = .GlobalEnv) 29 | assign("df", df, envir = .GlobalEnv) 30 | assign("df_fn", df_fn, envir = .GlobalEnv) 31 | }) 32 | 33 | 34 | test_that("cacheing flow works", { 35 | 36 | # pass #1 37 | write.csv(df, file_path, row.names = FALSE) 38 | eddy <- get_current_eddy() 39 | env <- new.env() 40 | df[, "Species"] <- as.character(df[, "Species"]) 41 | test_rflow <- 42 | file_path %>% 43 | flow_file_source() %>% 44 | flow_fn(stringsAsFactors = FALSE, fn = read.csv) %>% 45 | flow_dfg(1:3, fn = df_fn, group_by = "Species") %>% 46 | flow_dfr(fn = identity) %>% 47 | flow_ns_sink("collected_result", ns = env) 48 | 49 | expect_equal(length(eddy$flow_lst), 5) 50 | 51 | flow1 <- eddy$flow_lst[[1]] 52 | flow2 <- eddy$flow_lst[[2]] 53 | flow3 <- eddy$flow_lst[[3]] 54 | flow4 <- eddy$flow_lst[[4]] 55 | flow5 <- eddy$flow_lst[[5]] 56 | 57 | collected_flow1 <- flow1 %>% collect() 58 | collected_flow2 <- flow2 %>% collect() 59 | collected_flow3 <- flow3 %>% collect() 60 | collected_flow4 <- flow4 %>% collect() 61 | 62 | row_hash1 <- flow4$out_df[1, "..row_hash.."] 63 | expected_df <- df[1:3, , drop = FALSE] 64 | expected_df <- expected_df %>% 65 | dplyr::group_by(Species) %>% 66 | dplyr::mutate(rm = mean(Sepal.Length)) 67 | expected_df["Species"] <- as.character(expected_df["Species"]) 68 | 69 | expect_true(test_rflow$is_valid) 70 | expect_equal(nrow(env[["collected_result"]]), 3) 71 | expect_true(fs::is_absolute_path(collected_flow1)) 72 | expect_equal(nrow(collected_flow2), 20) 73 | expect_equal(nrow(flow3$out_df), 3) 74 | expect_equal(nrow(flow4$out_df), 3) 75 | 76 | # pass #2 77 | df[1, "Species"] <- "versicolor" 78 | write.csv(df, file_path, row.names = FALSE) 79 | 80 | eddy <- use_eddy("test_eddy2", cache = cache_fmem_test) 81 | test_rflow <- 82 | file_path %>% 83 | flow_file_source() %>% 84 | flow_fn(stringsAsFactors = FALSE, fn = read.csv) %>% 85 | flow_dfg(1:3, fn = df_fn, group_by = "Species") %>% 86 | flow_dfr(fn = identity) %>% 87 | flow_ns_sink("collected_result", ns = env) 88 | 89 | expect_equal(length(eddy$flow_lst), 5) 90 | 91 | flow1 <- eddy$flow_lst[[1]] 92 | flow2 <- eddy$flow_lst[[2]] 93 | flow3 <- eddy$flow_lst[[3]] 94 | flow4 <- eddy$flow_lst[[4]] 95 | 96 | collected_flow1 <- flow1 %>% collect() 97 | collected_flow2 <- flow2 %>% collect() 98 | collected_flow3 <- flow3 %>% collect() 99 | collected_flow4 <- flow4 %>% collect() 100 | 101 | row_hash2 <- flow4$out_df[2, "..row_hash.."] 102 | expect_false(row_hash1 == row_hash2) 103 | expect_equal(nrow(flow3$out_df), 6) 104 | expect_equal(nrow(flow4$out_df), 6) 105 | expect_equal(nrow(env[["collected_result"]]), 3) 106 | expect_true(fs::is_absolute_path(collected_flow1)) 107 | expect_equal(nrow(collected_flow2), 20) 108 | }) 109 | 110 | 111 | teardown({ 112 | get_current_eddy()$terminate() 113 | set_current_eddy("default_eddy") 114 | 115 | unlink(file_path, force = TRUE) 116 | unlink(cache_dir, recursive = TRUE, force = TRUE) 117 | 118 | base::rm(list = "cache_fmem_test", envir = .GlobalEnv) 119 | base::rm(list = "cache_dir", envir = .GlobalEnv) 120 | base::rm(list = "file_path", envir = .GlobalEnv) 121 | base::rm(list = "df", envir = .GlobalEnv) 122 | base::rm(list = "df_fn", envir = .GlobalEnv) 123 | }) 124 | -------------------------------------------------------------------------------- /tests/testthat/test-sink-ns.R: -------------------------------------------------------------------------------- 1 | # Tests for sink-ns ------------------------------------------------------------ 2 | context("tests for sink-ns") 3 | 4 | if (digest::digest(Sys.info()[-c(2, 3)]) %in% c( 5 | "2e85e2a3018ecf3b2e5fc03bfb20fd39" 6 | )) { 7 | skip("cache-memory-file functions") 8 | } 9 | 10 | 11 | setup({ 12 | test_fn <- function(x, y) { x + y } 13 | test_fn2 <- function(x, y) { x * y } 14 | test_fn3 <- function(x) {x} 15 | test_fn4 <- function(x, y) {list(x = x, y = y)} 16 | test_sink_var <- "value1" 17 | test_env <- new.env() 18 | 19 | assign("test_env", test_env, envir = .GlobalEnv) 20 | assign("test_sink_var", test_sink_var, envir = test_env) 21 | assign("test_fn", test_fn, envir = .GlobalEnv) 22 | assign("test_fn2", test_fn2, envir = .GlobalEnv) 23 | assign("test_fn3", test_fn3, envir = .GlobalEnv) 24 | assign("test_fn4", test_fn4, envir = .GlobalEnv) 25 | }) 26 | 27 | 28 | test_that("flow_ns_sink() works", { 29 | flow_sink <- flow_ns_sink("value2", "test_sink_var", test_env) 30 | 31 | expect_equal(flow_sink$fn_id, "test_sink_var") 32 | expect_equal(test_env[["test_sink_var"]], "value2") 33 | expect_equal(flow_sink$state_index, 1) 34 | 35 | flow_sink <- flow_ns_sink("value2", "test_sink_var", test_env) 36 | 37 | expect_equal(flow_sink$state_index, 1) 38 | }) 39 | 40 | 41 | test_that("flow_ns_sink() works with R6flow", { 42 | test_flow <- make_flow_fn(test_fn) 43 | test_rflow <- test_flow(1, 2) 44 | flow_sink2 <- flow_ns_sink(test_rflow, "test_sink_flow", test_env) 45 | 46 | expect_equal(flow_sink2$fn_id, "test_sink_flow") 47 | expect_equal(test_env[["test_sink_flow"]], 3) 48 | expect_equal(flow_sink2$state_index, 1) 49 | }) 50 | 51 | 52 | test_that("flow_ns_sink() works with multiple R6flow", { 53 | test_flow <- make_flow_fn(test_fn) 54 | test_rflow <- test_flow(1, 2) 55 | flow_sink2 <- flow_ns_sink(test_rflow, "test_sink_flow", test_env) 56 | 57 | expect_equal(flow_sink2$fn_id, "test_sink_flow") 58 | expect_equal(test_env[["test_sink_flow"]], 3) 59 | expect_equal(flow_sink2$state_index, 1) 60 | 61 | test_flow2 <- make_flow_fn(test_fn2) 62 | test_rflow2 <- test_flow2(1, 2) 63 | flow_sink2 <- flow_ns_sink(test_rflow2, "test_sink_flow", test_env) 64 | 65 | expect_equal(test_env[["test_sink_flow"]], 2) 66 | expect_equal(flow_sink2$state_index, 2) 67 | }) 68 | 69 | 70 | test_that("flow_ns_sink() works with flow_fn", { 71 | test_flow_fn <- flow_fn(1, 2, fn = test_fn) 72 | flow_sink3 <- flow_ns_sink(test_flow_fn, "test_sink_fn", test_env) 73 | 74 | expect_equal(flow_sink3$fn_id, "test_sink_fn") 75 | expect_equal(test_env[["test_sink_fn"]], 3) 76 | expect_equal(flow_sink3$state_index, 1) 77 | 78 | test_flow_fn2 <- flow_fn(1, 3, fn = test_fn) 79 | flow_sink3 <- flow_ns_sink(test_flow_fn2, "test_sink_fn", test_env) 80 | 81 | expect_equal(test_env[["test_sink_fn"]], 4) 82 | expect_equal(flow_sink3$state_index, 2) 83 | }) 84 | 85 | 86 | test_that("flow_ns_sink() works with NULL", { 87 | flow_sink_null <- flow_ns_sink(NULL, "test_sink_null", test_env) 88 | 89 | expect_equal(flow_sink_null$fn_id, "test_sink_null") 90 | expect_equal(test_env[["test_sink_null"]], NULL) 91 | expect_equal(flow_sink_null$state_index, 1) 92 | }) 93 | 94 | 95 | test_that("flow_ns_sink() works with NA", { 96 | flow_sink_na <- flow_ns_sink(NA, "test_sink_na", test_env) 97 | 98 | expect_equal(flow_sink_na$fn_id, "test_sink_na") 99 | expect_equal(test_env[["test_sink_na"]], NA) 100 | expect_equal(flow_sink_na$state_index, 1) 101 | }) 102 | 103 | 104 | test_that("flow_ns_sink() works with environment", { 105 | sink_env <- new.env() 106 | flow_sink_env <- flow_ns_sink(sink_env, "test_sink_env", test_env) 107 | 108 | expect_equal(flow_sink_env$fn_id, "test_sink_env") 109 | expect_equal(test_env[["test_sink_env"]], sink_env) 110 | expect_equal(flow_sink_env$state_index, 1) 111 | }) 112 | 113 | 114 | test_that("flow_ns_sink() stops with non existent variable", { 115 | expect_error( 116 | flow_sink <- flow_ns_sink(non_existent_var, "test_sink_var", test_env)) 117 | }) 118 | 119 | 120 | test_that("flow_ns_sink stops with non valid var_name", { 121 | expect_error( 122 | flow_sink <- flow_ns_sink("value1", 1, test_env)) 123 | expect_error( 124 | flow_sink <- flow_ns_sink("value1", TRUE, test_env)) 125 | expect_error( 126 | flow_sink <- flow_ns_sink("value1", NULL, test_env)) 127 | expect_error( 128 | flow_sink <- flow_ns_sink("value1", NA, test_env)) 129 | expect_error( 130 | flow_sink <- flow_ns_sink("value1", NA_character_, test_env)) 131 | expect_error( 132 | flow_sink <- flow_ns_sink("value1", character(), test_env)) 133 | expect_error( 134 | flow_sink <- flow_ns_sink("value1", c("var1", "var2"), test_env)) 135 | expect_error( 136 | flow_sink <- flow_ns_sink("value1", list(), test_env)) 137 | }) 138 | 139 | 140 | test_that("flow_ns_sink() stops with non environment", { 141 | non_environment <- function() {} 142 | expect_error( 143 | flow_sink <- flow_ns_sink("value1", "test_sink_var", non_environment)) 144 | }) 145 | 146 | 147 | teardown({ 148 | base::rm(list = "test_fn", envir = .GlobalEnv) 149 | base::rm(list = "test_fn2", envir = .GlobalEnv) 150 | base::rm(list = "test_fn3", envir = .GlobalEnv) 151 | base::rm(list = "test_fn4", envir = .GlobalEnv) 152 | base::rm(list = "test_env", envir = .GlobalEnv) 153 | }) 154 | -------------------------------------------------------------------------------- /tests/testthat/test-source-file.R: -------------------------------------------------------------------------------- 1 | # Tests for source file -------------------------------------------------------- 2 | context("tests for source-file") 3 | 4 | if (digest::digest(Sys.info()[-c(2, 3)]) %in% c( 5 | "2e85e2a3018ecf3b2e5fc03bfb20fd39" 6 | )) { 7 | skip("cache-memory-file functions") 8 | } 9 | 10 | setup({ 11 | file1 <- tempfile(pattern = "test-rflow-") 12 | 13 | df1 <- tibble::remove_rownames(head(mtcars)) 14 | df2 <- tibble::remove_rownames(tail(mtcars)) 15 | 16 | assign("file1", file1, envir = .GlobalEnv) 17 | assign("df1", df1, envir = .GlobalEnv) 18 | assign("df2", df2, envir = .GlobalEnv) 19 | }) 20 | 21 | 22 | test_that("flow_file_source() works", { 23 | 24 | get_current_eddy()$reset() 25 | 26 | write.csv(df1, file1, row.names = FALSE) 27 | 28 | file_path <- as.character(fs::path(file1)) 29 | test_rflow_source <- flow_file_source(file_path) 30 | 31 | expect_equal(test_rflow_source$state_index, 1L) 32 | }) 33 | 34 | 35 | test_that("flow_file_source() works when file modified", { 36 | 37 | get_current_eddy()$reset() 38 | 39 | write.csv(df1, file1, row.names = FALSE) 40 | 41 | file_path <- as.character(fs::path(file1)) 42 | test_rflow_source <- flow_file_source(file_path) 43 | 44 | expect_equal(test_rflow_source$state_index, 1L) 45 | 46 | write.csv(df2, file1, row.names = FALSE) 47 | test_rflow_source <- flow_file_source(file_path) 48 | 49 | expect_equal(test_rflow_source$state_index, 2L) 50 | }) 51 | 52 | 53 | test_that("flow_file_source() works when adding same file", { 54 | 55 | get_current_eddy()$reset() 56 | 57 | write.csv(df1, file1, row.names = FALSE) 58 | 59 | file_path <- as.character(fs::path(file1)) 60 | test_rflow_source <- flow_file_source(file_path) 61 | 62 | expect_equal(test_rflow_source$state_index, 1L) 63 | 64 | test_rflow_source <- flow_file_source(file_path) 65 | 66 | expect_equal(test_rflow_source$state_index, 1L) 67 | }) 68 | 69 | 70 | test_that("flow_file_source() works with fs::path type", { 71 | 72 | get_current_eddy()$reset() 73 | 74 | write.csv(df1, file1, row.names = FALSE) 75 | 76 | file_path <- fs::path(file1) 77 | expect_silent(test_rflow_source <- flow_file_source(file_path)) 78 | 79 | expect_equal(test_rflow_source$state_index, 1L) 80 | }) 81 | 82 | 83 | test_that("flow_file_source() works with non existent file path", { 84 | 85 | get_current_eddy()$reset() 86 | 87 | file_path <- as.character(fs::path("test", "path")) 88 | expect_silent(test_rflow_source <- flow_file_source(file_path)) 89 | }) 90 | 91 | 92 | test_that("flow_file_source() works when file present, missing, and changed", { 93 | 94 | get_current_eddy()$reset() 95 | 96 | write.csv(df1, file1, row.names = FALSE) 97 | 98 | file_path <- as.character(fs::path(file1)) 99 | test_rflow_source <- flow_file_source(file_path) 100 | 101 | # here there are already two states 102 | expect_equal(test_rflow_source$state_index, 1L) 103 | unlink(file1) 104 | 105 | test_rflow_source <- flow_file_source(file_path) 106 | expect_equal(test_rflow_source$state_index, 2L) 107 | 108 | write.csv(df2, file1, row.names = FALSE) 109 | test_rflow_source <- flow_file_source(file_path) 110 | 111 | expect_equal(test_rflow_source$state_index, 3L) 112 | expect_equal(NROW(test_rflow_source$state), 3) 113 | }) 114 | 115 | 116 | test_that("flow_file_source() stops with non valid input", { 117 | 118 | get_current_eddy()$reset() 119 | 120 | expect_error(test_rflow_source <- flow_file_source(1)) 121 | expect_error(test_rflow_source <- flow_file_source(TRUE)) 122 | expect_error(test_rflow_source <- flow_file_source(character())) 123 | expect_error(test_rflow_source <- flow_file_source(character())) 124 | expect_error(test_rflow_source <- flow_file_source(list())) 125 | expect_error(test_rflow_source <- flow_file_source(NULL)) 126 | expect_error(test_rflow_source <- flow_file_source(NA)) 127 | expect_error(test_rflow_source <- flow_file_source(NA_character_)) 128 | }) 129 | 130 | test_that("flow_file_source() works with 2 paths", { 131 | 132 | get_current_eddy()$reset() 133 | 134 | expect_silent(test_rflow_source <- flow_file_source(c("path1", "path2"))) 135 | expect_true(test_rflow_source$get_element("path1")$is_current) 136 | expect_true(test_rflow_source$get_element("path2")$is_current) 137 | }) 138 | 139 | 140 | teardown({ 141 | get_current_eddy()$reset() 142 | 143 | if (dir.exists(file1)) { 144 | unlink(file1, recursive = TRUE, force = TRUE) 145 | } 146 | 147 | base::rm(list = "file1", envir = .GlobalEnv) 148 | base::rm(list = "df1", envir = .GlobalEnv) 149 | base::rm(list = "df2", envir = .GlobalEnv) 150 | }) 151 | --------------------------------------------------------------------------------