├── .Rbuildignore ├── DESCRIPTION ├── LICENSE ├── LICENSE.md ├── NAMESPACE ├── R ├── clean_up.R ├── get_portalled_items.R ├── message.R ├── pull.R ├── purge.R └── push.R ├── README.md └── man ├── get_portalled_items.Rd ├── pull.Rd └── push.Rd /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^LICENSE\.md$ 2 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: portal 2 | Title: Move data between your local R sessions 3 | Version: 0.0.1 4 | Authors@R: 5 | person("Miles", "McBain", , "miles.mcbain@gmail.com", role = c("aut", "cre")) 6 | Description: A bit of whimsy that saves you creating a temporary file. 7 | License: MIT + file LICENSE 8 | Encoding: UTF-8 9 | Roxygen: list(markdown = TRUE) 10 | RoxygenNote: 7.1.2 11 | Imports: 12 | arrow, 13 | backports, 14 | fs, 15 | glue, 16 | qs, 17 | stats, 18 | utils 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | YEAR: 2022 2 | COPYRIGHT HOLDER: portal authors 3 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | Copyright (c) 2022 portal authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | export(get_portalled_items) 4 | export(pull) 5 | export(purge) 6 | export(push) 7 | -------------------------------------------------------------------------------- /R/clean_up.R: -------------------------------------------------------------------------------- 1 | .onLoad <- function(libname, pkgname) { 2 | backports::import(pkgname, "dir.exists") 3 | backports::import(pkgname, "R_user_dir", force = TRUE) 4 | pkg_user_dir <- get_pkg_user_dir() 5 | file_info_df <- fs::file_info(list.files(pkg_user_dir, full.names = TRUE)) 6 | # Remove all files from previous days 7 | old_files <- as.numeric(Sys.Date() - as.Date(file_info_df$birth_time)) > 0 8 | unlink(file_info_df[old_files,]$path) 9 | } 10 | -------------------------------------------------------------------------------- /R/get_portalled_items.R: -------------------------------------------------------------------------------- 1 | get_pkg_user_dir <- function() { 2 | portal_user_dir <- R_user_dir("portal") 3 | if (!dir.exists(portal_user_dir)) { 4 | dir.create(portal_user_dir, recursive = TRUE) 5 | } 6 | portal_user_dir 7 | } 8 | 9 | #' Catalogue all the objects in the abyss. 10 | #' 11 | #' Where you need a specfic object, knowing its name is necessary to conjur it. 12 | #' 13 | #' Names have power. 14 | #' 15 | #' @export 16 | get_portalled_items <- function() { 17 | items <- list.files(get_pkg_user_dir(), full.names = TRUE) 18 | items_info <- lapply(items, \(filename) { 19 | file_info <- file.info(filename) 20 | file_info$filename <- filename 21 | file_info 22 | }) 23 | item_dataframe <- do.call(rbind, items_info) 24 | 25 | # if the item_dataframe was NULL, we found no items. 26 | # return an empty dataframe of right shape and type for no items, 27 | # rather than NULL. 28 | if (is.null(item_dataframe)) { 29 | dummy_frame <- file.info("__not_a_real_file__") 30 | dummy_frame$filename <- NA_character_ 31 | item_dataframe <- stats::na.omit(dummy_frame) 32 | } 33 | item_dataframe 34 | } 35 | -------------------------------------------------------------------------------- /R/message.R: -------------------------------------------------------------------------------- 1 | 2 | in_message <- function(object_name) { 3 | in_portal <- 4 | c( 5 | " __", 6 | "| |---", 7 | "| |", 8 | "{object_name} -> | |", 9 | "| |", 10 | "|__|---" 11 | ) 12 | in_portal[[4]] <- glue::glue(in_portal[[4]]) 13 | in_portal[-4] <- paste0(strrep(" ", nchar(object_name) + nchar(" -> ")), in_portal[-4]) 14 | paste0(in_portal, collapse = "\n") |> 15 | cat("\n") 16 | } 17 | 18 | out_message <- function(object_name) { 19 | out_portal <- 20 | " 21 | __ 22 | ---| | 23 | | | 24 | | | -> {object_name} 25 | | | 26 | ---|__| 27 | 28 | " 29 | glue::glue(out_portal) |> 30 | cat("\n") 31 | } 32 | -------------------------------------------------------------------------------- /R/pull.R: -------------------------------------------------------------------------------- 1 | #' Conjur an object from the ether 2 | #' 3 | #' Oldest objects emerge first. 4 | #' 5 | #' @param object Search the abyss for an object that has this symbol's name. In 6 | #' the abyss not all searches end. 7 | #' @export 8 | pull <- function(object) { 9 | portal_items <- get_portalled_items() 10 | item_filename <- 11 | if (!missing(object)) { 12 | item_names <- 13 | fs::path_ext_remove(fs::path_file(portal_items$filename)) 14 | portal_items[item_names == as.character(substitute(object)), ]$filename 15 | } else { 16 | first_in <- utils::head(sort(portal_items$ctime), 1) 17 | portal_items[portal_items$ctime == first_in, ]$filename 18 | } 19 | if (length(item_filename) == 0) stop("Could not find item in the portal.") 20 | object_data <- read_item(item_filename) 21 | object_name <- fs::path_ext_remove(fs::path_file(item_filename)) 22 | assign(object_name, object_data, envir = .GlobalEnv) 23 | out_message(object_name) 24 | unlink(item_filename) 25 | } 26 | 27 | read_item <- function(filename) { 28 | extension <- fs::path_ext(filename) 29 | deserialiser <- switch( 30 | extension, 31 | "rds" = readRDS, 32 | "qs" = qs::qread, 33 | "parquet" = arrow::read_parquet, 34 | NULL 35 | ) 36 | if (is.null(deserialiser)) { 37 | stop("Portalled item has unsupported file extension: ", filename) 38 | } 39 | deserialiser(filename) 40 | } 41 | -------------------------------------------------------------------------------- /R/purge.R: -------------------------------------------------------------------------------- 1 | #' @export 2 | purge <- function(object = NULL) { 3 | portal_items <- get_portalled_items() 4 | if (!is.null(object)) { 5 | item_names <- 6 | fs::path_ext_remove(fs::path_file(portal_items$filename)) 7 | target_name <- 8 | portal_items[item_names == as.character(substitute(object)), ]$filename 9 | if (length(target_name) == 0) { 10 | stop( 11 | "Could not find object ", 12 | object, 13 | " in the portal" 14 | ) 15 | } 16 | unlink(target_name) 17 | } else { 18 | lapply(portal_items$filename, unlink) 19 | } 20 | invisible() 21 | } 22 | -------------------------------------------------------------------------------- /R/push.R: -------------------------------------------------------------------------------- 1 | #' Push an object into the ether 2 | #' 3 | #' @param object the object to be thrown into the portal. Beware traveller, 4 | #' somewhere in the abyss a file takes the symbol's name. 5 | #' @param serialiser what kind of file the object is backed by. "rds" (default), "qs", or "parquet" 6 | #' @export 7 | push <- function( 8 | object, 9 | serialiser = getOption("portal.serialser", default = "rds") 10 | ) { 11 | 12 | serialiser_fun <- switch( 13 | serialiser, 14 | "rds" = saveRDS, 15 | "qs" = qs::qsave, 16 | "parquet" = arrow::write_parquet, 17 | NULL 18 | ) 19 | if (is.null(serialiser_fun)) { 20 | stop( 21 | "Unknown serialiser option: ", 22 | serialiser, 23 | ". Choose 'rds', 'qs', or 'parquet'" 24 | ) 25 | } 26 | object_name <- substitute(object) 27 | if (!is.symbol(object_name)) { 28 | stop("The portal can only take symbols (objects), not expressions, assign your expressoin to an object?") 29 | } 30 | serialiser_fun( 31 | object, 32 | file.path( 33 | get_pkg_user_dir(), 34 | paste(as.character(object_name), serialiser, sep =".") 35 | ) 36 | ) 37 | in_message(as.character(object_name)) 38 | } 39 | 40 | function() { 41 | foo <- mtcars 42 | push(foo) 43 | get_portalled_items() 44 | pull() 45 | } 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # portal 3 | 4 | 5 | 6 | 7 | Move R datasets between local sessions. 8 | 9 | ## Installation 10 | 11 | You can install the development version of portal from [GitHub](https://github.com/) with: 12 | 13 | ``` r 14 | # install.packages("devtools") 15 | devtools::install_github("milesmcbain/portal") 16 | ``` 17 | 18 | ## Usage 19 | 20 | You discovered a problem in your pipeline. You've isolated the data but the code 21 | that's blowing up lives in one of your packages. You fire up the package project 22 | and prepare to get cracking... but geez wouldn't it be nice if you could just 23 | pull that dataset from your pipeline, directly into your package REPL? 24 | 25 | I mean reprexes are nice and all... but a dataset that creates the issue is right there... 26 | 27 | ## Pipeline R session 28 | 29 | ```R 30 | ❯ breaking_dataset |> portal::push() 31 | 32 | # __ 33 | # | |--- 34 | # | | 35 | # breaking_dataset -> | | 36 | # | | 37 | # |__|--- 38 | 39 | ``` 40 | 41 | ## Package session 42 | 43 | ```R 44 | ❯ portal::pull() 45 | # __ 46 | # ---| | 47 | # | | 48 | # | | -> breaking_dataset 49 | # | | 50 | # ---|__| 51 | 52 | ``` 53 | 54 | `breaking_dataset` is now in the global environment. Let's get fixing! 55 | 56 | ## Advanced usage 57 | 58 | By default datasets are serialised using Rds. If that's too slow `push` takes a `serialiser` argument which can be set as an option: `portal.serialser`. Supported choices are in `c("rds", "qs", "parquet")`. 59 | 60 | ## Problems 61 | 62 | `push()` uses the name of the symbol you passed in to name the file, which in 63 | turn informs `pull` what to call the object when it is imported. Weird object 64 | names won't work that well. Notably if you use this with the `{magrittr}` pipe 65 | you'll have a bad time, since the symbol (and file) will be named `.`. 66 | 67 | Similarly pushing unnamed expressions might result in invalid file names and so won't work. 68 | 69 | I'll probably fix these eventually. Contributions are welcome. 70 | -------------------------------------------------------------------------------- /man/get_portalled_items.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get_portalled_items.R 3 | \name{get_portalled_items} 4 | \alias{get_portalled_items} 5 | \title{Catalogue all the objects in the abyss.} 6 | \usage{ 7 | get_portalled_items() 8 | } 9 | \description{ 10 | Where you need a specfic object, knowing its name is necessary to conjur it. 11 | } 12 | \details{ 13 | Names have power. 14 | } 15 | -------------------------------------------------------------------------------- /man/pull.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/pull.R 3 | \name{pull} 4 | \alias{pull} 5 | \title{Conjur an object from the ether} 6 | \usage{ 7 | pull(object) 8 | } 9 | \arguments{ 10 | \item{object}{Search the abyss for an object that has this symbol's name. In 11 | the abyss not all searches end.} 12 | } 13 | \description{ 14 | Oldest objects emerge first. 15 | } 16 | -------------------------------------------------------------------------------- /man/push.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/push.R 3 | \name{push} 4 | \alias{push} 5 | \title{Push an object into the ether} 6 | \usage{ 7 | push(object, serialiser = getOption("portal.serialser", default = "rds")) 8 | } 9 | \arguments{ 10 | \item{object}{the object to be thrown into the portal. Beware traveller, 11 | somewhere in the abyss a file takes the symbol's name.} 12 | 13 | \item{serialiser}{what kind of file the object is backed by. "rds" (default), "qs", or "parquet"} 14 | } 15 | \description{ 16 | Push an object into the ether 17 | } 18 | --------------------------------------------------------------------------------