├── .Rbuildignore ├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── DESCRIPTION ├── Makefile ├── NAMESPACE ├── NEWS.md ├── R ├── colclass_dict.R ├── csvy-package.R ├── detect_metadata.R ├── get_yaml_header.R ├── read_csvy.R ├── read_metadata.R ├── write_csvy.R └── write_metadata.R ├── README.Rmd ├── README.md ├── appveyor.yml ├── docs ├── CONTRIBUTING.html ├── ISSUE_TEMPLATE.html ├── PULL_REQUEST_TEMPLATE.html ├── authors.html ├── index.html ├── jquery.sticky-kit.min.js ├── link.svg ├── news │ └── index.html ├── pkgdown.css ├── pkgdown.js ├── pkgdown.yml └── reference │ ├── csvy.html │ ├── index.html │ ├── read_csvy.html │ └── write_csvy.html ├── inst ├── CITATION └── examples │ ├── example1.csvy │ └── example2.csvy ├── man ├── colclass_dict.Rd ├── csvy.Rd ├── get_yaml_header.Rd ├── read_csvy.Rd ├── read_metadata.Rd ├── write_csvy.Rd └── write_metadata.Rd └── tests ├── test-all.R └── testthat ├── test-metadata.R ├── test-read_csvy.R ├── test-write_csvy.R └── test-write_metadata.R /.Rbuildignore: -------------------------------------------------------------------------------- 1 | .github/* 2 | ^\.travis\.yml$ 3 | ^appveyor\.yml$ 4 | ^travis-tool\.sh$ 5 | ^Makefile$ 6 | ^README\.Rmd$ 7 | ^README\.html$ 8 | ^README_files$ 9 | ^README_files/.+$ 10 | ^CONTRIBUTING\.md$ 11 | ^inst/standarderrors\.pdf$ 12 | ^figure$ 13 | ^figure/.+$ 14 | ^cache/.+$ 15 | ^docs$ 16 | ^docs/.+$ 17 | ^ignore$ 18 | ^inst/doc/.+\.log$ 19 | ^inst/doc/.+\.Rmd$ 20 | ^vignettes/figure$ 21 | ^vignettes/figure/.+$ 22 | ^vignettes/.+\.aux$ 23 | ^vignettes/.+\.bbl$ 24 | ^vignettes/.+\.blg$ 25 | ^vignettes/.+\.dvi$ 26 | ^vignettes/.+\.log$ 27 | ^vignettes/.+\.out$ 28 | ^vignettes/.+\.pdf$ 29 | ^vignettes/.+\.sty$ 30 | ^vignettes/.+\.tex$ 31 | ^data-raw$ 32 | ^revdep$ 33 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributions to **csvy** are welcome from anyone and are best sent as pull requests on [the GitHub repository](https://github.com/leeper/csvy/). This page provides some instructions to potential contributors about how to add to the package. 2 | 3 | 1. Contributions can be submitted as [a pull request](https://help.github.com/articles/creating-a-pull-request/) on GitHub by forking or cloning the [repo](https://github.com/leeper/csvy/), making changes and submitting the pull request. 4 | 5 | 2. Pull requests should involve only one commit per substantive change. This means if you change multiple files (e.g., code and documentation), these changes should be committed together. If you don't know how to do this (e.g., you are making changes in the GitHub web interface) just submit anyway and the maintainer will clean things up. 6 | 7 | 3. All contributions must be submitted consistent with the package license ([GPL-2](http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)). 8 | 9 | 4. All contributions need to be noted in the `Authors@R` field in the [DESCRIPTION](https://github.com/leeper/csvy/blob/master/DESCRIPTION). Just follow the format of the existing entries to add your name (and, optionally, email address). Substantial contributions should also be noted in [`inst/CITATION`](https://github.com/leeper/csvy/blob/master/inst/CITATION). 10 | 11 | 5. Please run `R CMD BUILD csvy` and `R CMD CHECK csvy_VERSION.tar.gz` before submitting the pull request to check for any errors. 12 | 13 | Some specific types of changes that you might make are: 14 | 15 | 1. Documentation-only changes (e.g., to Rd files, README, vignettes). This is great! All contributions are welcome. 16 | 17 | 2. Changes requiring a new package dependency should be discussed on the GitHub issues page before submitting a pull request. 18 | 19 | 3. Message translations. These are very appreciated! The format is a pain, but if you're doing this I'm assuming you're already familiar with it. 20 | 21 | Any questions you have can be opened as GitHub issues or directed to thosjleeper (at) gmail.com. 22 | 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please specify whether your issue is about: 2 | 3 | - [ ] a possible bug 4 | - [ ] a question about package functionality 5 | - [ ] a suggested code or documentation change, improvement to the code, or feature request 6 | 7 | If you are reporting (1) a bug or (2) a question about code, please supply: 8 | 9 | - [a fully reproducible example](http://stackoverflow.com/questions/5963269/how-to-make-a-great-r-reproducible-example) using a publicly available dataset (or provide your data) 10 | - if an error is occurring, include the output of `traceback()` run immediately after the error occurs 11 | - the output of `sessionInfo()` 12 | 13 | Put your code here: 14 | 15 | ```R 16 | ## load package 17 | library("csvy") 18 | 19 | ## code goes here 20 | 21 | 22 | ## session info for your system 23 | sessionInfo() 24 | ``` 25 | 26 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please ensure the following before submitting a PR: 2 | 3 | - [ ] if suggesting code changes or improvements, [open an issue](https://github.com/leeper/csvy/issues/new) first 4 | - [ ] for all but trivial changes (e.g., typo fixes), add your name to [DESCRIPTION](https://github.com/leeper/csvy/blob/master/DESCRIPTION) 5 | - [ ] for all but trivial changes (e.g., typo fixes), documentation your change in [NEWS.md](https://github.com/leeper/csvy/blob/master/NEWS.md) with a parenthetical reference to the issue number being addressed 6 | - [ ] if changing documentation, edit files in `/R` not `/man` and run `devtools::document()` to update documentation 7 | - [ ] add code or new test files to [`/tests`](https://github.com/leeper/csvy/tree/master/tests/testthat) for any new functionality or bug fix 8 | - [ ] make sure `R CMD check` runs without error before submitting the PR 9 | 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | .Rhistory 3 | iris.csvy 4 | revdep/* 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: r 2 | sudo: false 3 | r_packages: 4 | - covr 5 | after_success: 6 | - Rscript -e 'library("covr");codecov()' 7 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: csvy 2 | Type: Package 3 | Title: Import and Export CSV Data with a YAML Metadata Header 4 | Version: 0.3.0 5 | Date: 2018-07-31 6 | Authors@R: c(person("Thomas J.", "Leeper", 7 | role = c("aut", "cre"), 8 | email = "thosjleeper@gmail.com", 9 | comment = c(ORCID = "0000-0003-4097-6326")), 10 | person("Alexey N.", "Shiklomanov", 11 | role = c("aut"), 12 | email = "alexey.shiklomanov@gmail.com", 13 | comment = c(ORCID = "0000-0003-4022-5979")), 14 | person("Jonathan", "Carroll", 15 | email = "rpkg@jcarroll.com.au", 16 | role = c("aut"), 17 | comment = c(ORCID = "0000-0002-1404-5264")) 18 | ) 19 | Description: Support for import from and export to the CSVY file format. CSVY is a file format that combines the simplicity of CSV (comma-separated values) with the metadata of other plain text and binary formats (JSON, XML, Stata, etc.) by placing a YAML header on top of a regular CSV. 20 | URL: https://github.com/leeper/csvy 21 | BugReports: https://github.com/leeper/csvy/issues 22 | Imports: 23 | tools, 24 | data.table, 25 | jsonlite, 26 | yaml 27 | Suggests: 28 | testthat, 29 | datasets 30 | License: GPL-2 31 | RoxygenNote: 6.0.1 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | pkg = $(shell basename $(CURDIR)) 2 | 3 | all: build 4 | 5 | NAMESPACE: R/* 6 | Rscript -e "devtools::document()" 7 | 8 | README.md: README.Rmd 9 | Rscript -e "knitr::knit('README.Rmd')" 10 | 11 | README.html: README.md 12 | pandoc -o README.html README.md 13 | 14 | ../$(pkg)*.tar.gz: DESCRIPTION NAMESPACE README.md 15 | cd ../ && R CMD build $(pkg) 16 | 17 | build: ../$(pkg)*.tar.gz 18 | 19 | check: ../$(pkg)*.tar.gz 20 | cd ../ && R CMD check $(pkg)*.tar.gz 21 | rm ../$(pkg)*.tar.gz 22 | 23 | revdep: ../$(pkg)*.tar.gz 24 | Rscript -e "devtools::revdep_check()" 25 | 26 | install: ../$(pkg)*.tar.gz 27 | cd ../ && R CMD INSTALL $(pkg)*.tar.gz 28 | rm ../$(pkg)*.tar.gz 29 | 30 | website: R/* README.md DESCRIPTION 31 | Rscript -e "pkgdown::build_site()" 32 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | export(get_yaml_header) 4 | export(read_csvy) 5 | export(read_metadata) 6 | export(write_csvy) 7 | export(write_metadata) 8 | importFrom(data.table,fread) 9 | importFrom(data.table,fwrite) 10 | importFrom(jsonlite,fromJSON) 11 | importFrom(jsonlite,write_json) 12 | importFrom(stats,setNames) 13 | importFrom(tools,file_ext) 14 | importFrom(yaml,as.yaml) 15 | importFrom(yaml,yaml.load) 16 | -------------------------------------------------------------------------------- /NEWS.md: -------------------------------------------------------------------------------- 1 | # csvy 0.3.0 2 | 3 | - Updated support to current CSVY specifications. (#13, h/t Michael Chirico) 4 | - Argument `sep2` in `write_csvy()` has been corrected to `dec`. 5 | - Fixed an unclosed connection bug. (#23) 6 | 7 | # csvy 0.2.2 8 | 9 | - If reading a file `data.csvy` without a metadata header, and a `data.[yaml|yml|json]` file is present (in the same directory), that will be automatically read-in as the metadata (completes requests for #10, h/t @jonocarroll) 10 | 11 | # csvy 0.2.1 12 | 13 | * Expanded test suite and fixed some small bugs in the process. 14 | * Parse YAML header file first, then pass column classes to `data.table::fread` to improve performance (#9, Alexey Shiklomanov) 15 | 16 | # csvy 0.2.0 17 | 18 | - Removed support for `utils::read.csv()` and `readr::read_csv()` for simplicity. 19 | - Updated support to current CSVY specifications. (#13, h/t Michael Chirico) 20 | - Substantially changed internal code and added markup. 21 | - Changed example files. 22 | - Added option to output metadata to separate YAML or JSON file. (#10, h/t Hadley Wickham) 23 | 24 | # csvy 0.1.2 25 | 26 | - Address header that is not in the same order as data columns. (#1) 27 | - Support for `readr::read_csv()` and `utils::read.csv()`. (#2) 28 | 29 | # csvy 0.1.1 30 | 31 | - Initial release 32 | -------------------------------------------------------------------------------- /R/colclass_dict.R: -------------------------------------------------------------------------------- 1 | #' Dictionary of column classes for reading data 2 | colclass_dict <- c( 3 | "string" = "character", 4 | "integer" = "integer", 5 | "number" = "numeric", 6 | "factor" = "character", # Convert to factor afterwards -- fread doesn't do factors 7 | "date" = "Date", 8 | "datetime" = "POSIXct", 9 | "boolean" = "logical" 10 | ) 11 | -------------------------------------------------------------------------------- /R/csvy-package.R: -------------------------------------------------------------------------------- 1 | #' @name csvy 2 | #' @docType package 3 | #' @aliases csvy csvy-package 4 | #' @title Import and Export CSV Data With a YAML Metadata Header 5 | #' @description CSVY is a file format that combines the simplicity of CSV (comma-separated values) with the metadata of other plain text and binary formats (JSON, XML, Stata, etc.). The \href{http://csvy.org/}{CSVY file specification} is simple: place a YAML header on top of a regular CSV. The csvy package implements this format using two functions: \code{\link{write_csvy}} and \code{\link{read_csvy}}. 6 | NULL 7 | -------------------------------------------------------------------------------- /R/detect_metadata.R: -------------------------------------------------------------------------------- 1 | detect_metadata <- function(file) { 2 | filedir <- dirname(file) 3 | possible_metadata <- dir(filedir, pattern = "\\.[Jj]{1}[Ss]{1}[Oo]{1}[Nn]{1}$|\\.[Yy]{1}[Aa]?[Mm]{1}[Ll]{1}", full.names = TRUE) 4 | if (length(possible_metadata) > 1) { 5 | # too many potential metadata files found 6 | stop("More than one yaml/yml/json files detected in same directory as data") 7 | } else if (length(possible_metadata) == 0) { 8 | # no metadata file found, so just read file 9 | return(NULL) 10 | } 11 | # one file found 12 | message(sprintf("Attempting to read metadata from auto-detected file: %s", basename(possible_metadata))) 13 | return(read_metadata(possible_metadata)) 14 | } 15 | -------------------------------------------------------------------------------- /R/get_yaml_header.R: -------------------------------------------------------------------------------- 1 | #' Retrieve YAML header from file 2 | #' 3 | #' Note that this assumes only one Yaml header, starting on the first line of the file. 4 | #' 5 | #' @inheritParams read_csvy 6 | #' @param yaml_rxp Regular expression for parsing YAML header 7 | #' @param verbose Logical. If \code{TRUE}, print warning if no header found. 8 | #' @return Character vector of lines containing YAML header, or `NULL` if no YAML header found. 9 | #' @export 10 | get_yaml_header <- function(file, yaml_rxp = "^\\#*---[[:space:]]*$", verbose = TRUE) { 11 | # read first line to check for header 12 | con <- file(file, "r") 13 | on.exit(close(con)) 14 | first_line <- readLines(con, n = 1L) 15 | if (!length(first_line) || !grepl(yaml_rxp, first_line)) { 16 | if (isTRUE(verbose)) { 17 | warning("No YAML header found.") 18 | } 19 | return(NULL) 20 | } 21 | 22 | # if header, read it in until "---" found 23 | iline <- 1L 24 | closing_tag <- FALSE 25 | out <- character() 26 | while (!isTRUE(closing_tag)) { 27 | out[iline] <- readLines(con, n = 1L) 28 | if (grepl(yaml_rxp, out[iline])) { 29 | closing_tag <- TRUE 30 | } else { 31 | iline <- iline + 1L 32 | } 33 | } 34 | 35 | # remove leading comment character, if present 36 | if (all(grepl("^#", out))) { 37 | out <- gsub("^#", "", out) 38 | } 39 | return(out) 40 | } 41 | -------------------------------------------------------------------------------- /R/read_csvy.R: -------------------------------------------------------------------------------- 1 | #' @title Import CSVY data 2 | #' @description Import CSVY data as a data.frame 3 | #' @param file A character string or R connection specifying a file. 4 | #' @param metadata Optionally, a character string specifying a YAML (\dQuote{.yaml}) or JSON (\dQuote{.json}) file containing metadata (in lieu of including it in the header of the file). 5 | #' @param stringsAsFactors A logical specifying whether to treat character columns as factors. Passed to \code{\link[utils]{read.csv}} or \code{\link[data.table]{fread}} depending on the value of \code{method}. Ignored for \code{method = 'readr'} which never returns factors. 6 | #' @param detect_metadata A logical specifying whether to auto-detect a metadata file if none is specified (and if no header is found). 7 | #' @param \dots Additional arguments passed to \code{\link[data.table]{fread}}. 8 | #' @examples 9 | #' read_csvy(system.file("examples", "example1.csvy", package = "csvy")) 10 | #' 11 | #' @importFrom tools file_ext 12 | #' @importFrom jsonlite fromJSON 13 | #' @importFrom data.table fread 14 | #' @importFrom yaml yaml.load 15 | #' @export 16 | #' @seealso \code{\link{write_csvy}} 17 | read_csvy <- 18 | function( 19 | file, 20 | metadata = NULL, 21 | stringsAsFactors = FALSE, 22 | detect_metadata = TRUE, 23 | ... 24 | ) { 25 | 26 | # setup factor coercion conditional on presence of 'levels' metadata field 27 | if (isTRUE(stringsAsFactors)) { 28 | try_to_factorize <- "always" 29 | } else if (stringsAsFactors == "conditional") { 30 | stringsAsFactors <- FALSE 31 | try_to_factorize <- "conditional" 32 | } else { 33 | try_to_factorize <- "never" 34 | } 35 | 36 | if (is.null(metadata)) { 37 | metadata_raw <- get_yaml_header(file, verbose = FALSE) 38 | if (is.null(metadata_raw) & !detect_metadata) { 39 | # no metadata found in file and no auto-detection requested 40 | message("No metadata header found. Reading file as CSV.") 41 | out <- data.table::fread(input = file, sep = "auto", header = "auto", 42 | stringsAsFactors = stringsAsFactors, 43 | data.table = FALSE, ...) 44 | return(out) 45 | } else if (is.null(metadata_raw) & isTRUE(detect_metadata)) { 46 | # no metadata found in file but auto-detection requested 47 | message("No metadata header found in file, so attempting to auto-detect metadata file.") 48 | skip_lines <- 0L 49 | metadata_list <- detect_metadata(file) 50 | if (is.null(metadata_list)) { 51 | message("No metadata file found. Reading file as CSV.") 52 | out <- data.table::fread(input = file, sep = "auto", header = "auto", 53 | stringsAsFactors = stringsAsFactors, 54 | data.table = FALSE, ...) 55 | return(out) 56 | } 57 | } else { 58 | # metadata found in file 59 | skip_lines <- length(metadata_raw) + 1L # Including opening and closing "---" 60 | metadata_list <- yaml::yaml.load(paste(metadata_raw, collapse = "\n")) 61 | } 62 | } else { 63 | skip_lines <- 0L 64 | metadata_list <- read_metadata(metadata) 65 | } 66 | 67 | # find variable-level metadata 'fields' 68 | if ("fields" %in% names(metadata_list)) { 69 | fields <- metadata_list$fields 70 | col_classes <- NULL 71 | } else if ("schema" %in% names(metadata_list)) { 72 | fields <- metadata_list$schema$fields 73 | field_types <- vapply(fields, "[[", character(1), "type") 74 | col_classes <- colclass_dict[field_types] 75 | names(col_classes) <- vapply(fields, "[[", character(1), "name") 76 | } else { 77 | fields <- NULL 78 | col_classes <- NULL 79 | } 80 | 81 | # find 'dialect' to use for importing, if available 82 | if ("dialect" %in% names(metadata_list)) { 83 | ## delimiter 84 | sep <- metadata_list$dialect$delimeter 85 | if (is.null(sep)) sep <- "auto" 86 | ## header 87 | header <- as.logical(metadata_list$dialect$header) 88 | if (is.null(header)) { 89 | header <- "auto" 90 | } 91 | ## there are other args here but we really don't need them 92 | ## need to decide how to use them 93 | } else { 94 | sep <- "auto" 95 | header <- "auto" 96 | } 97 | 98 | # load the data 99 | out <- data.table::fread( 100 | file = file, 101 | sep = sep, 102 | header = header, 103 | stringsAsFactors = stringsAsFactors, 104 | data.table = FALSE, 105 | colClasses = col_classes, 106 | skip = skip_lines, 107 | ... 108 | ) 109 | 110 | # add data frame-level metadata to data 111 | out <- add_dataset_metadata(data_frame = out, metadata_list = metadata_list) 112 | 113 | # add variable-level metadata to data 114 | out <- add_variable_metadata(data = out, fields = fields, try_to_factorize = try_to_factorize) 115 | 116 | return(out) 117 | } 118 | 119 | check_variable_metadata <- function(data, fields) { 120 | if (is.null(fields)) { 121 | return(NULL) 122 | } 123 | 124 | hnames <- lapply(fields, `[[`, "name") 125 | 126 | missing_from_metadata <- names(data)[!names(data) %in% hnames] 127 | if (length(missing_from_metadata)) { 128 | warning("Metadata is missing for ", 129 | ngettext(length(missing_from_metadata), "variable", "variables"), 130 | " listed in data: ", paste(missing_from_metadata, collapse = ", ")) 131 | } 132 | 133 | missing_from_data <- unlist(hnames)[!unlist(hnames) %in% names(data)] 134 | if (length(missing_from_data)) { 135 | warning("Data is missing for ", 136 | ngettext(length(missing_from_data), "variable", "variables"), 137 | " listed in frontmatter: ", paste(missing_from_metadata, collapse = ", ")) 138 | } 139 | 140 | duplicated_metadata <- unlist(hnames)[duplicated(unlist(hnames))] 141 | if (length(duplicated_metadata)) { 142 | warning("Duplicate metadata entries for ", 143 | ngettext(length(duplicated_metadata), "variable", "variables"), 144 | " listed in frontmatter: ", paste(duplicated_metadata, collapse = ", ")) 145 | } 146 | 147 | duplicated_columns <- unlist(hnames)[duplicated(unlist(hnames))] 148 | if (length(duplicated_columns)) { 149 | warning("Duplicate column names for ", 150 | ngettext(length(duplicated_columns), "variable", "variables"), 151 | ": ", paste(duplicated_metadata, collapse = ", ")) 152 | } 153 | 154 | NULL 155 | } 156 | 157 | add_variable_metadata <- function(data, fields, try_to_factorize = "never") { 158 | 159 | # check metadata against header row 160 | check_variable_metadata(data = data, fields = fields) 161 | 162 | # add metadata to data, iterating across metadata list 163 | metadata_names <- lapply(fields, `[[`, "name") 164 | for (i in seq_along(fields)) { 165 | # grab attributes for this variable 166 | fields_this_col <- fields[[i]] 167 | 168 | # add 'title' field 169 | if ("title" %in% names(fields_this_col)) { 170 | attr(data[[i]], "label") <- fields_this_col[["title"]] 171 | } 172 | # add 'description' field 173 | if ("description" %in% names(fields_this_col)) { 174 | attr(data[[i]], "description") <- fields_this_col[["description"]] 175 | } 176 | ## store attributes already calculated 177 | dat_attributes <- attributes(data[[i]]) 178 | 179 | # handle 'type' and 'format' fields 180 | ## 'type' 181 | if ("type" %in% names(fields_this_col)) { 182 | if (fields_this_col[["type"]] == "string") { 183 | ## character/factor 184 | if (try_to_factorize == "always") { 185 | # convert all character to factor 186 | if (is.null(fields_this_col[["levels"]])) { 187 | try(data[[i]] <- as.factor(data[[i]])) 188 | } else { 189 | try(data[[i]] <- factor(data[[i]], levels = fields_this_col[["levels"]])) 190 | } 191 | } else if (try_to_factorize == "conditional") { 192 | # convert character to factor if levels are present 193 | if (is.null(fields_this_col[["levels"]])) { 194 | try(data[[i]] <- as.character(data[[i]])) 195 | } else { 196 | try(data[[i]] <- factor(data[[i]], levels = fields_this_col[["levels"]])) 197 | } 198 | } else { 199 | # do not convert character to factor 200 | try(data[[i]] <- as.character(data[[i]])) 201 | } 202 | } else if (fields_this_col[["type"]] == "date") { 203 | try(data[[i]] <- as.Date(data[[i]])) 204 | } else if (fields_this_col[["type"]] == "datetime") { 205 | try(data[[i]] <- as.POSIXct(data[[i]])) 206 | } else if (fields_this_col[["type"]] == "boolean") { 207 | try(data[[i]] <- as.logical(data[[i]])) 208 | } else if (fields_this_col[["type"]] == "number") { 209 | try(data[[i]] <- as.numeric(data[[i]])) 210 | } 211 | ## replace attributes 212 | attributes(data[[i]]) <- dat_attributes 213 | } 214 | ## 'format' (just added as an attribute for now) 215 | if ("format" %in% names(fields_this_col)) { 216 | attr(data[[i]], "format") <- fields_this_col[["format"]] 217 | } 218 | ## add 'levels' (if not added above during factor coercion) 219 | if ("levels" %in% names(fields_this_col) && (!"levels" %in% attributes(data[[i]]))) { 220 | attr(data[[i]], "levels") <- fields_this_col[["levels"]] 221 | } 222 | ## add 'labels' (not in schema but useful) 223 | if ("labels" %in% names(fields_this_col) && (!"labels" %in% attributes(data[[i]]))) { 224 | attr(data[[i]], "labels") <- fields_this_col[["labels"]] 225 | } 226 | rm(fields_this_col) 227 | } 228 | 229 | return(data) 230 | } 231 | 232 | add_dataset_metadata <- function(data_frame, metadata_list) { 233 | if ("profile" %in% names(metadata_list)) { 234 | attr(data_frame, "profile") <- metadata_list[["profile"]] 235 | } 236 | if ("title" %in% names(metadata_list)) { 237 | attr(data_frame, "title") <- metadata_list[["title"]] 238 | } 239 | if ("description" %in% names(metadata_list)) { 240 | attr(data_frame, "description") <- metadata_list[["description"]] 241 | } 242 | if ("name" %in% names(metadata_list)) { 243 | attr(data_frame, "name") <- metadata_list[["name"]] 244 | } 245 | if ("format" %in% names(metadata_list)) { 246 | attr(data_frame, "format") <- metadata_list[["format"]] 247 | } 248 | if ("sources" %in% names(metadata_list)) { 249 | attr(data_frame, "sources") <- metadata_list[["sources"]] 250 | } 251 | if ("licenses" %in% names(metadata_list)) { 252 | attr(data_frame, "sources") <- metadata_list[["licenses"]] 253 | } 254 | return(data_frame) 255 | } 256 | -------------------------------------------------------------------------------- /R/read_metadata.R: -------------------------------------------------------------------------------- 1 | #' @title Read metadata 2 | #' @md 3 | #' @description Read csvy metadata from an external `.yml/.yaml` or `.json` file 4 | #' 5 | #' @param file full path of file from which to read the metadata. 6 | #' 7 | #' @return the metadata as a list 8 | #' 9 | #' @importFrom yaml yaml.load 10 | #' @importFrom jsonlite fromJSON 11 | #' @importFrom tools file_ext 12 | #' 13 | #' @export 14 | read_metadata <- function(file) { 15 | ext <- tolower(tools::file_ext(file)) 16 | if (ext %in% c("yaml", "yml")) { 17 | metadata_list <- yaml::yaml.load(paste(readLines(file), collapse = "\n")) 18 | } else if (ext == "json") { 19 | metadata_list <- jsonlite::fromJSON(file, simplifyDataFrame = FALSE) 20 | } else { 21 | stop("'metadata' should be either a .json or .yaml file.") ## should fail 22 | } 23 | return(metadata_list) 24 | } 25 | -------------------------------------------------------------------------------- /R/write_csvy.R: -------------------------------------------------------------------------------- 1 | #' @title Export CSVY data 2 | #' @description Export data.frame to CSVY 3 | #' @param x A data.frame. 4 | #' @param file A character string or R connection specifying a file. 5 | #' @param metadata Optionally, a character string specifying a YAML (\dQuote{.yaml}) or JSON (\dQuote{.json}) file to write the metadata (in lieu of including it in the header of the file). 6 | #' @param sep A character string specifying a between-field separator. Passed to \code{\link[data.table]{fwrite}}. 7 | #' @param dec A character string specifying a within-field separator. Passed to \code{\link[data.table]{fwrite}}. 8 | #' @param comment_header A logical indicating whether to comment the lines containing the YAML front matter. Default is \code{TRUE}. 9 | #' @param metadata_only A logical indicating whether only the metadata should be produced (no CSV component). 10 | #' @param name A character string specifying a name for the dataset. 11 | #' @param \dots Additional arguments passed to \code{\link[data.table]{fwrite}}. 12 | #' @examples 13 | #' library("datasets") 14 | #' write_csvy(head(iris)) 15 | #' 16 | #' # write yaml w/o comment charaters 17 | #' write_csvy(head(iris), comment_header = FALSE) 18 | #' 19 | #' @importFrom stats setNames 20 | #' @importFrom data.table fwrite 21 | #' @importFrom yaml as.yaml 22 | #' @importFrom jsonlite write_json 23 | #' @export 24 | #' @seealso \code{\link{write_csvy}} 25 | write_csvy <- 26 | function( 27 | x, 28 | file, 29 | metadata = NULL, 30 | sep = ",", 31 | dec = ".", 32 | comment_header = if (is.null(metadata)) TRUE else FALSE, 33 | name = deparse(substitute(x)), 34 | metadata_only = FALSE, 35 | ... 36 | ) { 37 | 38 | ## data-level metadata 39 | metadata_list <- list(profile = "tabular-data-package", 40 | name = name) 41 | att <- attributes(x) 42 | metadata_list <- c(metadata_list, att[!names(att) %in% c("names", "class", "row.names")]) 43 | 44 | ## build variable-specific metadata list 45 | fields <- list() 46 | for (i in seq_along(x)) { 47 | # grab attributes for this variable 48 | fields_this_col <- attributes(x[[i]]) 49 | 50 | # initialize metadata list for this variable 51 | fields[[i]] <- list() 52 | 53 | # add 'name' field 54 | fields[[i]][["name"]] <- names(x)[i] 55 | # add 'title' field 56 | if ("label" %in% names(fields_this_col)) { 57 | fields[[i]][["title"]] <- fields_this_col[["label"]] 58 | } 59 | # R has no canonical analogue to 'description' field, but if it's there add it 60 | if ("description" %in% names(fields_this_col)) { 61 | fields[[i]][["description"]] <- fields_this_col[["description"]] 62 | } 63 | # add 'type' field 64 | ## default is 'string' unless specified otherwise 65 | fields[[i]][["type"]] <- switch(class(x[[i]])[1L], 66 | character = "string", 67 | Date = "date", 68 | integer = "integer", 69 | logical = "boolean", 70 | numeric = "number", 71 | POSIXct = "datetime", 72 | "string") 73 | if ("labels" %in% names(fields_this_col)) { 74 | fields[[i]][["labels"]] <- 75 | setNames(as.list(unname(fields_this_col$labels)), names(fields_this_col$labels)) 76 | } 77 | if ("levels" %in% names(fields_this_col)) { 78 | fields[[i]][["levels"]] <- 79 | setNames(as.list(unname(fields_this_col$levels)), names(fields_this_col$levels)) 80 | } 81 | rm(fields_this_col) 82 | } 83 | metadata_list[["fields"]] <- fields 84 | 85 | if (!is.null(metadata)) { 86 | ## write metadata to separate file 87 | write_metadata(metadata_list, metadata) 88 | ## don't write the csv component if metadata_only is TRUE 89 | if (!metadata_only) { 90 | # write CSV 91 | data.table::fwrite(x = x, file = file, sep = sep, dec = dec, ...) 92 | } 93 | } else { 94 | # write metadata to file 95 | y <- paste0("---\n", yaml::as.yaml(metadata_list), "---\n") 96 | if (isTRUE(comment_header)){ 97 | con <- textConnection(y) 98 | on.exit(close(con)) 99 | m <- readLines(con) 100 | y <- paste0("#", m[-length(m)],collapse = "\n") 101 | y <- c(y, "\n") 102 | } 103 | # write data to file 104 | if (missing(file)) { 105 | cat(y) 106 | data.table::fwrite(x = x, file = "", sep = sep, dec = dec, append = TRUE, col.names = TRUE, ...) 107 | } else { 108 | cat(y, file = file) 109 | # append CSV to file 110 | data.table::fwrite(x = x, file = file, sep = sep, dec = dec, append = TRUE, col.names = TRUE, ...) 111 | } 112 | } 113 | invisible(x) 114 | } 115 | -------------------------------------------------------------------------------- /R/write_metadata.R: -------------------------------------------------------------------------------- 1 | #' @title Write csvy metadata 2 | #' @md 3 | #' @description Write csvy metadata to an external `.yml/.yaml` or `.json` file 4 | #' 5 | #' @param metadata_list metadata to be stored. Must be valid as per 6 | #' [yaml::as.yaml()] or [jsonlite::write_json()] for that particular output 7 | #' type. 8 | #' @param file full path of file in which to save the metadata. 9 | #' 10 | #' @importFrom yaml as.yaml 11 | #' @importFrom jsonlite write_json 12 | #' @importFrom tools file_ext 13 | #' 14 | #' @return `NULL` (invisibly) 15 | #' @export 16 | write_metadata <- function(metadata_list = NULL, file = NULL) { 17 | 18 | if (is.null(metadata_list) || !is.list(metadata_list)) stop("must provide metadata_list as a list") 19 | if (is.null(file) || !is.character(file)) stop("metadata (filename) must be provided") 20 | 21 | ## get file extension 22 | ext <- tolower(tools::file_ext(file)) 23 | 24 | # write metadata to separate metadata file 25 | if (ext %in% c("yml", "yaml")) { 26 | cat(yaml::as.yaml(metadata_list), file = file) 27 | } else if (ext == "json") { 28 | jsonlite::write_json(metadata_list, path = file) 29 | } else { 30 | warning("'metadata' should be either a .json or .yaml file.") ## TODO stop? 31 | } 32 | 33 | return(invisible(NULL)) 34 | 35 | } 36 | -------------------------------------------------------------------------------- /README.Rmd: -------------------------------------------------------------------------------- 1 | # Import and Export CSV Data With a YAML Metadata Header 2 | 3 | CSVY is a file format that combines the simplicity of CSV (comma-separated values) with the metadata of other plain text and binary formats (JSON, XML, Stata, etc.). The [CSVY file specification](http://csvy.org/) is simple: place a YAML header on top of a regular CSV. The yaml header is formatted according to the [Table Schema](https://frictionlessdata.io/specs/table-schema/) of a [Tabular Data Package](https://frictionlessdata.io/specs/tabular-data-package/). 4 | 5 | A CSVY file looks like this: 6 | 7 | ``` 8 | #--- 9 | #profile: tabular-data-resource 10 | #name: my-dataset 11 | #path: https://raw.githubusercontent.com/csvy/csvy.github.io/master/examples/example.csvy 12 | #title: Example file of csvy 13 | #description: Show a csvy sample file. 14 | #format: csvy 15 | #mediatype: text/vnd.yaml 16 | #encoding: utf-8 17 | #schema: 18 | # fields: 19 | # - name: var1 20 | # type: string 21 | # - name: var2 22 | # type: integer 23 | # - name: var3 24 | # type: number 25 | #dialect: 26 | # csvddfVersion: 1.0 27 | # delimiter: "," 28 | # doubleQuote: false 29 | # lineTerminator: "\r\n" 30 | # quoteChar: "\"" 31 | # skipInitialSpace: true 32 | # header: true 33 | #sources: 34 | #- title: The csvy specifications 35 | # path: http://csvy.org/ 36 | # email: '' 37 | #licenses: 38 | #- name: CC-BY-4.0 39 | # title: Creative Commons Attribution 4.0 40 | # path: https://creativecommons.org/licenses/by/4.0/ 41 | #--- 42 | var1,var2,var3 43 | A,1,2.0 44 | B,3,4.3 45 | ``` 46 | 47 | Which we can read into R like this: 48 | 49 | 50 | ```{r} 51 | library("csvy") 52 | str(read_csvy(system.file("examples", "example1.csvy", package = "csvy"))) 53 | ``` 54 | 55 | Optional comment characters on the YAML lines make the data readable with any standard CSV parser while retaining the ability to import and export variable- and file-level metadata. The CSVY specification does not use these, but the csvy package for R does so that you (and other users) can continue to rely on `utils::read.csv()` or `readr::read_csv()` as usual. The `import()` function in [rio](https://cran.r-project.org/package=rio) supports CSVY natively. 56 | 57 | ### Export 58 | 59 | To create a CSVY file from R, just do: 60 | 61 | ```{r} 62 | library("csvy") 63 | library("datasets") 64 | write_csvy(iris, "iris.csvy") 65 | ``` 66 | 67 | It is also possible to export the metadata to separate YAML or JSON file (and then also possible to import from those separate files) by specifying the `metadata` field in `write_csvy()` and `read_csvy()`. 68 | 69 | ### Import 70 | 71 | To read a CSVY into R, just do: 72 | 73 | ```{r} 74 | d1 <- read_csvy("iris.csvy") 75 | str(d1) 76 | ``` 77 | 78 | or use any other appropriate data import function to ignore the YAML metadata: 79 | 80 | ```{r} 81 | d2 <- utils::read.table("iris.csvy", sep = ",", header = TRUE) 82 | str(d2) 83 | ``` 84 | 85 | ```{r, echo=FALSE} 86 | unlink("iris.csvy") 87 | ``` 88 | 89 | ## Package Installation 90 | 91 | The package is available on [CRAN](https://cran.r-project.org/package=csvy) and can be installed directly in R using: 92 | 93 | ```R 94 | install.packages("csvy") 95 | ``` 96 | 97 | The latest development version on GitHub can be installed using **devtools**: 98 | 99 | ```R 100 | if(!require("remotes")){ 101 | install.packages("remotes") 102 | } 103 | remotes::install_github("leeper/csvy") 104 | ``` 105 | 106 | [![CRAN Version](http://www.r-pkg.org/badges/version/csvy)](https://cran.r-project.org/package=csvy) 107 | ![Downloads](http://cranlogs.r-pkg.org/badges/csvy) 108 | [![Travis-CI Build Status](https://travis-ci.org/leeper/csvy.png?branch=master)](https://travis-ci.org/leeper/csvy) 109 | [![Appveyor Build status](https://ci.appveyor.com/api/projects/status/sgttgdfcql63578u?svg=true)](https://ci.appveyor.com/project/leeper/csvy) 110 | [![codecov.io](http://codecov.io/github/leeper/csvy/coverage.svg?branch=master)](http://codecov.io/github/leeper/csvy?branch=master) 111 | 112 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Import and Export CSV Data With a YAML Metadata Header 2 | 3 | CSVY is a file format that combines the simplicity of CSV (comma-separated values) with the metadata of other plain text and binary formats (JSON, XML, Stata, etc.). The [CSVY file specification](http://csvy.org/) is simple: place a YAML header on top of a regular CSV. The yaml header is formatted according to the [Table Schema](https://frictionlessdata.io/specs/table-schema/) of a [Tabular Data Package](https://frictionlessdata.io/specs/tabular-data-package/). 4 | 5 | A CSVY file looks like this: 6 | 7 | ``` 8 | #--- 9 | #profile: tabular-data-resource 10 | #name: my-dataset 11 | #path: https://raw.githubusercontent.com/csvy/csvy.github.io/master/examples/example.csvy 12 | #title: Example file of csvy 13 | #description: Show a csvy sample file. 14 | #format: csvy 15 | #mediatype: text/vnd.yaml 16 | #encoding: utf-8 17 | #schema: 18 | # fields: 19 | # - name: var1 20 | # type: string 21 | # - name: var2 22 | # type: integer 23 | # - name: var3 24 | # type: number 25 | #dialect: 26 | # csvddfVersion: 1.0 27 | # delimiter: "," 28 | # doubleQuote: false 29 | # lineTerminator: "\r\n" 30 | # quoteChar: "\"" 31 | # skipInitialSpace: true 32 | # header: true 33 | #sources: 34 | #- title: The csvy specifications 35 | # path: http://csvy.org/ 36 | # email: '' 37 | #licenses: 38 | #- name: CC-BY-4.0 39 | # title: Creative Commons Attribution 4.0 40 | # path: https://creativecommons.org/licenses/by/4.0/ 41 | #--- 42 | var1,var2,var3 43 | A,1,2.0 44 | B,3,4.3 45 | ``` 46 | 47 | Which we can read into R like this: 48 | 49 | 50 | 51 | ```r 52 | library("csvy") 53 | str(read_csvy(system.file("examples", "example1.csvy", package = "csvy"))) 54 | ``` 55 | 56 | ``` 57 | ## 'data.frame': 2 obs. of 3 variables: 58 | ## $ var1: chr "A" "B" 59 | ## $ var2: int 1 3 60 | ## $ var3: num 2 4.3 61 | ## - attr(*, "profile")= chr "tabular-data-resource" 62 | ## - attr(*, "title")= chr "Example file of csvy" 63 | ## - attr(*, "description")= chr "Show a csvy sample file." 64 | ## - attr(*, "name")= chr "my-dataset" 65 | ## - attr(*, "format")= chr "csvy" 66 | ## - attr(*, "sources")=List of 1 67 | ## ..$ :List of 3 68 | ## .. ..$ name : chr "CC-BY-4.0" 69 | ## .. ..$ title: chr "Creative Commons Attribution 4.0" 70 | ## .. ..$ path : chr "https://creativecommons.org/licenses/by/4.0/" 71 | ``` 72 | 73 | Optional comment characters on the YAML lines make the data readable with any standard CSV parser while retaining the ability to import and export variable- and file-level metadata. The CSVY specification does not use these, but the csvy package for R does so that you (and other users) can continue to rely on `utils::read.csv()` or `readr::read_csv()` as usual. The `import()` function in [rio](https://cran.r-project.org/package=rio) supports CSVY natively. 74 | 75 | ### Export 76 | 77 | To create a CSVY file from R, just do: 78 | 79 | 80 | ```r 81 | library("csvy") 82 | library("datasets") 83 | write_csvy(iris, "iris.csvy") 84 | ``` 85 | 86 | It is also possible to export the metadata to separate YAML or JSON file (and then also possible to import from those separate files) by specifying the `metadata` field in `write_csvy()` and `read_csvy()`. 87 | 88 | ### Import 89 | 90 | To read a CSVY into R, just do: 91 | 92 | 93 | ```r 94 | d1 <- read_csvy("iris.csvy") 95 | str(d1) 96 | ``` 97 | 98 | ``` 99 | ## 'data.frame': 150 obs. of 5 variables: 100 | ## $ Sepal.Length: num 5.1 4.9 4.7 4.6 5 5.4 4.6 5 4.4 4.9 ... 101 | ## $ Sepal.Width : num 3.5 3 3.2 3.1 3.6 3.9 3.4 3.4 2.9 3.1 ... 102 | ## $ Petal.Length: num 1.4 1.4 1.3 1.5 1.4 1.7 1.4 1.5 1.4 1.5 ... 103 | ## $ Petal.Width : num 0.2 0.2 0.2 0.2 0.2 0.4 0.3 0.2 0.2 0.1 ... 104 | ## $ Species : chr "setosa" "setosa" "setosa" "setosa" ... 105 | ## ..- attr(*, "levels")= chr "setosa" "versicolor" "virginica" 106 | ## - attr(*, "profile")= chr "tabular-data-package" 107 | ## - attr(*, "name")= chr "iris" 108 | ``` 109 | 110 | or use any other appropriate data import function to ignore the YAML metadata: 111 | 112 | 113 | ```r 114 | d2 <- utils::read.table("iris.csvy", sep = ",", header = TRUE) 115 | str(d2) 116 | ``` 117 | 118 | ``` 119 | ## 'data.frame': 150 obs. of 5 variables: 120 | ## $ Sepal.Length: num 5.1 4.9 4.7 4.6 5 5.4 4.6 5 4.4 4.9 ... 121 | ## $ Sepal.Width : num 3.5 3 3.2 3.1 3.6 3.9 3.4 3.4 2.9 3.1 ... 122 | ## $ Petal.Length: num 1.4 1.4 1.3 1.5 1.4 1.7 1.4 1.5 1.4 1.5 ... 123 | ## $ Petal.Width : num 0.2 0.2 0.2 0.2 0.2 0.4 0.3 0.2 0.2 0.1 ... 124 | ## $ Species : Factor w/ 3 levels "setosa","versicolor",..: 1 1 1 1 1 1 1 1 1 1 ... 125 | ``` 126 | 127 | 128 | 129 | ## Package Installation 130 | 131 | The package is available on [CRAN](https://cran.r-project.org/package=csvy) and can be installed directly in R using: 132 | 133 | ```R 134 | install.packages("csvy") 135 | ``` 136 | 137 | The latest development version on GitHub can be installed using **devtools**: 138 | 139 | ```R 140 | if(!require("remotes")){ 141 | install.packages("remotes") 142 | } 143 | remotes::install_github("leeper/csvy") 144 | ``` 145 | 146 | [![CRAN Version](http://www.r-pkg.org/badges/version/csvy)](https://cran.r-project.org/package=csvy) 147 | ![Downloads](http://cranlogs.r-pkg.org/badges/csvy) 148 | [![Travis-CI Build Status](https://travis-ci.org/leeper/csvy.png?branch=master)](https://travis-ci.org/leeper/csvy) 149 | [![Appveyor Build status](https://ci.appveyor.com/api/projects/status/sgttgdfcql63578u?svg=true)](https://ci.appveyor.com/project/leeper/csvy) 150 | [![codecov.io](http://codecov.io/github/leeper/csvy/coverage.svg?branch=master)](http://codecov.io/github/leeper/csvy?branch=master) 151 | 152 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | # Download script file from GitHub 2 | init: 3 | ps: | 4 | $ErrorActionPreference = "Stop" 5 | Invoke-WebRequest http://raw.github.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1" 6 | Import-Module '..\appveyor-tool.ps1' 7 | 8 | environment: 9 | global: 10 | USE_RTOOLS: true 11 | 12 | install: 13 | ps: Bootstrap 14 | 15 | build_script: 16 | - travis-tool.sh install_deps 17 | 18 | test_script: 19 | - travis-tool.sh run_tests 20 | 21 | artifacts: 22 | - path: '*.Rcheck\**\*.log' 23 | name: Logs 24 | 25 | - path: '*.Rcheck\**\*.out' 26 | name: Logs 27 | 28 | - path: '*.Rcheck\**\*.fail' 29 | name: Logs 30 | 31 | - path: '*.Rcheck\**\*.Rout' 32 | name: Logs 33 | 34 | - path: '\*_*.tar.gz' 35 | name: Bits 36 | 37 | - path: '\*_*.zip' 38 | name: Bits 39 | -------------------------------------------------------------------------------- /docs/CONTRIBUTING.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | NA • csvy 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 36 | 37 | 38 | 39 | 40 | 41 |
42 |
43 | 80 | 81 | 82 |
83 | 84 |
85 |
86 |
87 |

NA

88 |
89 | 90 | 91 |

Contributions to csvy are welcome from anyone and are best sent as pull requests on the GitHub repository. This page provides some instructions to potential contributors about how to add to the package.

92 |
    93 |

  1. Contributions can be submitted as a pull request on GitHub by forking or cloning the repo, making changes and submitting the pull request.

    2. 94 |

  2. Pull requests should involve only one commit per substantive change. This means if you change multiple files (e.g., code and documentation), these changes should be committed together. If you don’t know how to do this (e.g., you are making changes in the GitHub web interface) just submit anyway and the maintainer will clean things up.

    3. 95 |

  3. All contributions must be submitted consistent with the package license (GPL-2).

    4. 96 |

  4. All contributions need to be noted in the Authors@R field in the DESCRIPTION. Just follow the format of the existing entries to add your name (and, optionally, email address). Substantial contributions should also be noted in inst/CITATION.

    5. 97 |

  5. Please run R CMD BUILD csvy and R CMD CHECK csvy_VERSION.tar.gz before submitting the pull request to check for any errors.

    6. 98 |
99 |

Some specific types of changes that you might make are:

100 |
    101 |

  1. Documentation-only changes (e.g., to Rd files, README, vignettes). This is great! All contributions are welcome.

    2. 102 |

  2. Changes requiring a new package dependency should be discussed on the GitHub issues page before submitting a pull request.

    3. 103 |

  3. Message translations. These are very appreciated! The format is a pain, but if you’re doing this I’m assuming you’re already familiar with it.

    4. 104 |
105 |

Any questions you have can be opened as GitHub issues or directed to thosjleeper (at) gmail.com.

106 | 107 | 108 |
109 | 110 |
111 | 112 | 113 | 123 |
124 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /docs/ISSUE_TEMPLATE.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | NA • csvy 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 36 | 37 | 38 | 39 | 40 | 41 |
42 |
43 | 80 | 81 | 82 |
83 | 84 |
85 |
86 |
87 |

NA

88 |
89 | 90 | 91 |

Please specify whether your issue is about:

92 |
    93 |
  • [ ] a possible bug
    • 94 |
  • [ ] a question about package functionality
    • 95 |
  • [ ] a suggested code or documentation change, improvement to the code, or feature request
    • 96 |
97 |

If you are reporting (1) a bug or (2) a question about code, please supply:

98 |
    99 |
  • 100 | a fully reproducible example using a publicly available dataset (or provide your data)
    • 101 |
  • if an error is occurring, include the output of traceback() run immediately after the error occurs
    • 102 |
  • the output of sessionInfo() 103 |
    • 104 |
105 |

Put your code here:

106 | 
## load package
107 | library("csvy")
108 | 
109 | ## code goes here
110 | 
111 | 
112 | ## session info for your system
113 | sessionInfo()
114 | 115 | 116 |
117 | 118 |
119 | 120 | 121 | 131 |
132 | 133 | 134 | 135 | 136 | -------------------------------------------------------------------------------- /docs/PULL_REQUEST_TEMPLATE.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | NA • csvy 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 36 | 37 | 38 | 39 | 40 | 41 |
42 |
43 | 80 | 81 | 82 |
83 | 84 |
85 |
86 |
87 |

NA

88 |
89 | 90 | 91 |

Please ensure the following before submitting a PR:

92 |
    93 |
  • [ ] if suggesting code changes or improvements, open an issue first
    • 94 |
  • [ ] for all but trivial changes (e.g., typo fixes), add your name to DESCRIPTION 95 |
    • 96 |
  • [ ] for all but trivial changes (e.g., typo fixes), documentation your change in NEWS.md with a parenthetical reference to the issue number being addressed
    • 97 |
  • [ ] if changing documentation, edit files in /R not /man and run devtools::document() to update documentation
    • 98 |
  • [ ] add code or new test files to /tests for any new functionality or bug fix
    • 99 |
  • [ ] make sure R CMD check runs without error before submitting the PR
    • 100 |
101 | 102 | 103 |
104 | 105 |
106 | 107 | 108 | 118 |
119 | 120 | 121 | 122 | 123 | -------------------------------------------------------------------------------- /docs/authors.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Citation and Authors • csvy 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 36 | 37 | 38 | 39 | 40 | 41 |
42 |
43 | 80 | 81 | 82 |
83 | 84 |
85 |
86 | 87 |
88 |

Citation

89 |
90 | 91 |

Leeper TJ (2018). 92 | csvy: Import and Export CSV Data with a YAML Metadata Header. 93 | R package version 0.2.1. 94 |

95 | 
@Manual{,
 96 |   title = {csvy: Import and Export CSV Data with a YAML Metadata Header},
 97 |   author = {Thomas J. Leeper},
 98 |   year = {2018},
 99 |   note = {R package version 0.2.1},
100 | }
101 |
102 |

Authors

103 |
104 | 105 |
    106 |
  • 107 |

    Thomas J. Leeper. Author, maintainer. 108 |

    109 |
    • 110 |
111 | 112 |
113 | 114 |
115 | 116 | 117 | 127 |
128 | 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Import and Export CSV Data with a YAML Metadata Header • csvy 9 | 10 | 11 | 12 | 13 | 14 | 15 | 19 | 20 | 21 |
22 |
60 | 61 | 62 | 63 |
64 |
65 | 66 | 67 | 68 | 69 |
70 |
71 |

72 | Import and Export CSV Data With a YAML Metadata Header

73 |

CSVY is a file format that combines the simplicity of CSV (comma-separated values) with the metadata of other plain text and binary formats (JSON, XML, Stata, etc.). The CSVY file specification is simple: place a YAML header on top of a regular CSV. The yaml header is formatted according to the Table Schema of a Tabular Data Package.

74 |

A CSVY file looks like this:

75 | 
#---
 76 | #name: my-dataset
 77 | #resources:
 78 | #- order: 1
 79 | #  schema:
 80 | #    fields:
 81 | #    - name: var1
 82 | #      type: string
 83 | #    - name: var2
 84 | #      type: integer
 85 | #    - name: var3
 86 | #      type: number
 87 | #  dialect:
 88 | #    csvddfVersion: 1.0
 89 | #    delimiter: ","
 90 | #    doubleQuote: false
 91 | #    lineTerminator: "\r\n"
 92 | #    quoteChar: "\""
 93 | #    skipInitialSpace: true
 94 | #    header: true
 95 | ---
 96 | var1,var2,var3
 97 | A,1,2.0
 98 | B,3,4.3
99 |

Which we can read into R like this:

100 | 
library("csvy")
101 | str(read_csvy(system.file("examples", "example3.csvy", package = "csvy")))
102 | 
## 'data.frame':    2 obs. of  3 variables:
103 | ##  $ var1: chr  "A" "B"
104 | ##  $ var2: int  1 3
105 | ##  $ var3: num  2 4.3
106 |

Optional comment characters on the YAML lines make the data readable with any standard CSV parser while retaining the ability to import and export variable- and file-level metadata. The CSVY specification does not use these, but the csvy package for R does so that you (and other users) can continue to rely on utils::read.csv() or readr::read_csv() as usual. The import() function in rio supports CSVY natively.

107 |
108 |

109 | Export

110 |

To create a CSVY file from R, just do:

111 | 
library("csvy")
112 | library("datasets")
113 | write_csvy(iris, "iris.csvy")
114 |

It is also possible to export the metadata to separate YAML or JSON file (and then also possible to import from those separate files) by specifying the metadata field in write_csvy() and read_csvy().

115 |
116 |
117 |

118 | Import

119 |

To read a CSVY into R, just do:

120 | 
d1 <- read_csvy("iris.csvy")
121 | str(d1)
122 | 
## 'data.frame':    150 obs. of  5 variables:
123 | ##  $ Sepal.Length: num  5.1 4.9 4.7 4.6 5 5.4 4.6 5 4.4 4.9 ...
124 | ##  $ Sepal.Width : num  3.5 3 3.2 3.1 3.6 3.9 3.4 3.4 2.9 3.1 ...
125 | ##  $ Petal.Length: num  1.4 1.4 1.3 1.5 1.4 1.7 1.4 1.5 1.4 1.5 ...
126 | ##  $ Petal.Width : num  0.2 0.2 0.2 0.2 0.2 0.4 0.3 0.2 0.2 0.1 ...
127 | ##  $ Species     : atomic  setosa setosa setosa setosa ...
128 | ##   ..- attr(*, "levels")= chr  "setosa" "versicolor" "virginica"
129 |

or use any other appropriate data import function to ignore the YAML metadata:

130 | 
d2 <- utils::read.table("iris.csvy", sep = ",", header = TRUE)
131 | str(d2)
132 | 
## 'data.frame':    150 obs. of  5 variables:
133 | ##  $ Sepal.Length: num  5.1 4.9 4.7 4.6 5 5.4 4.6 5 4.4 4.9 ...
134 | ##  $ Sepal.Width : num  3.5 3 3.2 3.1 3.6 3.9 3.4 3.4 2.9 3.1 ...
135 | ##  $ Petal.Length: num  1.4 1.4 1.3 1.5 1.4 1.7 1.4 1.5 1.4 1.5 ...
136 | ##  $ Petal.Width : num  0.2 0.2 0.2 0.2 0.2 0.4 0.3 0.2 0.2 0.1 ...
137 | ##  $ Species     : Factor w/ 3 levels "setosa","versicolor",..: 1 1 1 1 1 1 1 1 1 1 ...
138 |
139 |
140 |

141 | Package Installation

142 |

The package is available on CRAN and can be installed directly in R using:

143 | 
install.packages("csvy")
144 |

The latest development version on GitHub can be installed using devtools:

145 | 
if(!require("ghit")){
146 |     install.packages("ghit")
147 | }
148 | ghit::install_github("leeper/csvy")
149 |

CRAN Version DownloadsTravis-CI Build Status Appveyor Build status codecov.io

150 |
151 |
152 |
153 |
154 | 155 | 177 | 178 |
179 | 180 | 181 | 190 |
191 | 192 | 193 | 194 | -------------------------------------------------------------------------------- /docs/jquery.sticky-kit.min.js: -------------------------------------------------------------------------------- 1 | /* 2 | Sticky-kit v1.1.2 | WTFPL | Leaf Corcoran 2015 | http://leafo.net 3 | */ 4 | (function(){var b,f;b=this.jQuery||window.jQuery;f=b(window);b.fn.stick_in_parent=function(d){var A,w,J,n,B,K,p,q,k,E,t;null==d&&(d={});t=d.sticky_class;B=d.inner_scrolling;E=d.recalc_every;k=d.parent;q=d.offset_top;p=d.spacer;w=d.bottoming;null==q&&(q=0);null==k&&(k=void 0);null==B&&(B=!0);null==t&&(t="is_stuck");A=b(document);null==w&&(w=!0);J=function(a,d,n,C,F,u,r,G){var v,H,m,D,I,c,g,x,y,z,h,l;if(!a.data("sticky_kit")){a.data("sticky_kit",!0);I=A.height();g=a.parent();null!=k&&(g=g.closest(k)); 5 | if(!g.length)throw"failed to find stick parent";v=m=!1;(h=null!=p?p&&a.closest(p):b("
"))&&h.css("position",a.css("position"));x=function(){var c,f,e;if(!G&&(I=A.height(),c=parseInt(g.css("border-top-width"),10),f=parseInt(g.css("padding-top"),10),d=parseInt(g.css("padding-bottom"),10),n=g.offset().top+c+f,C=g.height(),m&&(v=m=!1,null==p&&(a.insertAfter(h),h.detach()),a.css({position:"",top:"",width:"",bottom:""}).removeClass(t),e=!0),F=a.offset().top-(parseInt(a.css("margin-top"),10)||0)-q, 6 | u=a.outerHeight(!0),r=a.css("float"),h&&h.css({width:a.outerWidth(!0),height:u,display:a.css("display"),"vertical-align":a.css("vertical-align"),"float":r}),e))return l()};x();if(u!==C)return D=void 0,c=q,z=E,l=function(){var b,l,e,k;if(!G&&(e=!1,null!=z&&(--z,0>=z&&(z=E,x(),e=!0)),e||A.height()===I||x(),e=f.scrollTop(),null!=D&&(l=e-D),D=e,m?(w&&(k=e+u+c>C+n,v&&!k&&(v=!1,a.css({position:"fixed",bottom:"",top:c}).trigger("sticky_kit:unbottom"))),eb&&!v&&(c-=l,c=Math.max(b-u,c),c=Math.min(q,c),m&&a.css({top:c+"px"})))):e>F&&(m=!0,b={position:"fixed",top:c},b.width="border-box"===a.css("box-sizing")?a.outerWidth()+"px":a.width()+"px",a.css(b).addClass(t),null==p&&(a.after(h),"left"!==r&&"right"!==r||h.append(a)),a.trigger("sticky_kit:stick")),m&&w&&(null==k&&(k=e+u+c>C+n),!v&&k)))return v=!0,"static"===g.css("position")&&g.css({position:"relative"}), 8 | a.css({position:"absolute",bottom:d,top:"auto"}).trigger("sticky_kit:bottom")},y=function(){x();return l()},H=function(){G=!0;f.off("touchmove",l);f.off("scroll",l);f.off("resize",y);b(document.body).off("sticky_kit:recalc",y);a.off("sticky_kit:detach",H);a.removeData("sticky_kit");a.css({position:"",bottom:"",top:"",width:""});g.position("position","");if(m)return null==p&&("left"!==r&&"right"!==r||a.insertAfter(h),h.remove()),a.removeClass(t)},f.on("touchmove",l),f.on("scroll",l),f.on("resize", 9 | y),b(document.body).on("sticky_kit:recalc",y),a.on("sticky_kit:detach",H),setTimeout(l,0)}};n=0;for(K=this.length;n 2 | 3 | 5 | 8 | 12 | 13 | -------------------------------------------------------------------------------- /docs/news/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Changelog • csvy 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 36 | 37 | 38 | 39 | 40 | 41 |
42 |
43 | 80 | 81 | 82 |
83 | 84 |
85 | 86 |
87 |
88 |

Change log

89 | Source: NEWS.md 90 |
91 | 92 |
93 |
94 |
95 | 96 | 103 | 104 |
105 | 106 |
107 |
108 |

Developed by Thomas J. Leeper.

109 |
110 | 111 |
112 |

Site built with pkgdown.

113 |
114 | 115 |
116 |
117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /docs/pkgdown.css: -------------------------------------------------------------------------------- 1 | /* Sticky footer */ 2 | 3 | /** 4 | * Basic idea: https://philipwalton.github.io/solved-by-flexbox/demos/sticky-footer/ 5 | * Details: https://github.com/philipwalton/solved-by-flexbox/blob/master/assets/css/components/site.css 6 | * 7 | * .Site -> body > .container 8 | * .Site-content -> body > .container .row 9 | * .footer -> footer 10 | * 11 | * Key idea seems to be to ensure that .container and __all its parents__ 12 | * have height set to 100% 13 | * 14 | */ 15 | 16 | html, body { 17 | height: 100%; 18 | } 19 | 20 | body > .container { 21 | display: flex; 22 | height: 100%; 23 | flex-direction: column; 24 | 25 | padding-top: 60px; 26 | } 27 | 28 | body > .container .row { 29 | flex: 1 0 auto; 30 | } 31 | 32 | footer { 33 | margin-top: 45px; 34 | padding: 35px 0 36px; 35 | border-top: 1px solid #e5e5e5; 36 | color: #666; 37 | display: flex; 38 | flex-shrink: 0; 39 | } 40 | footer p { 41 | margin-bottom: 0; 42 | } 43 | footer div { 44 | flex: 1; 45 | } 46 | footer .pkgdown { 47 | text-align: right; 48 | } 49 | footer p { 50 | margin-bottom: 0; 51 | } 52 | 53 | img.icon { 54 | float: right; 55 | } 56 | 57 | img { 58 | max-width: 100%; 59 | } 60 | 61 | /* Typographic tweaking ---------------------------------*/ 62 | 63 | .contents h1.page-header { 64 | margin-top: calc(-60px + 1em); 65 | } 66 | 67 | /* Section anchors ---------------------------------*/ 68 | 69 | a.anchor { 70 | margin-left: -30px; 71 | display:inline-block; 72 | width: 30px; 73 | height: 30px; 74 | visibility: hidden; 75 | 76 | background-image: url(./link.svg); 77 | background-repeat: no-repeat; 78 | background-size: 20px 20px; 79 | background-position: center center; 80 | } 81 | 82 | .hasAnchor:hover a.anchor { 83 | visibility: visible; 84 | } 85 | 86 | @media (max-width: 767px) { 87 | .hasAnchor:hover a.anchor { 88 | visibility: hidden; 89 | } 90 | } 91 | 92 | 93 | /* Fixes for fixed navbar --------------------------*/ 94 | 95 | .contents h1, .contents h2, .contents h3, .contents h4 { 96 | padding-top: 60px; 97 | margin-top: -60px; 98 | } 99 | 100 | /* Static header placement on mobile devices */ 101 | @media (max-width: 767px) { 102 | .navbar-fixed-top { 103 | position: absolute; 104 | } 105 | .navbar { 106 | padding: 0; 107 | } 108 | } 109 | 110 | 111 | /* Sidebar --------------------------*/ 112 | 113 | #sidebar { 114 | margin-top: 30px; 115 | } 116 | #sidebar h2 { 117 | font-size: 1.5em; 118 | margin-top: 1em; 119 | } 120 | 121 | #sidebar h2:first-child { 122 | margin-top: 0; 123 | } 124 | 125 | #sidebar .list-unstyled li { 126 | margin-bottom: 0.5em; 127 | } 128 | 129 | .orcid { 130 | height: 16px; 131 | vertical-align: middle; 132 | } 133 | 134 | /* Reference index & topics ----------------------------------------------- */ 135 | 136 | .ref-index th {font-weight: normal;} 137 | .ref-index h2 {font-size: 20px;} 138 | 139 | .ref-index td {vertical-align: top;} 140 | .ref-index .alias {width: 40%;} 141 | .ref-index .title {width: 60%;} 142 | 143 | .ref-index .alias {width: 40%;} 144 | .ref-index .title {width: 60%;} 145 | 146 | .ref-arguments th {text-align: right; padding-right: 10px;} 147 | .ref-arguments th, .ref-arguments td {vertical-align: top;} 148 | .ref-arguments .name {width: 20%;} 149 | .ref-arguments .desc {width: 80%;} 150 | 151 | /* Nice scrolling for wide elements --------------------------------------- */ 152 | 153 | table { 154 | display: block; 155 | overflow: auto; 156 | } 157 | 158 | /* Syntax highlighting ---------------------------------------------------- */ 159 | 160 | pre { 161 | word-wrap: normal; 162 | word-break: normal; 163 | border: 1px solid #eee; 164 | } 165 | 166 | pre, code { 167 | background-color: #f8f8f8; 168 | color: #333; 169 | } 170 | 171 | pre code { 172 | overflow: auto; 173 | word-wrap: normal; 174 | white-space: pre; 175 | } 176 | 177 | pre .img { 178 | margin: 5px 0; 179 | } 180 | 181 | pre .img img { 182 | background-color: #fff; 183 | display: block; 184 | height: auto; 185 | } 186 | 187 | code a, pre a { 188 | color: #375f84; 189 | } 190 | 191 | a.sourceLine:hover { 192 | text-decoration: none; 193 | } 194 | 195 | .fl {color: #1514b5;} 196 | .fu {color: #000000;} /* function */ 197 | .ch,.st {color: #036a07;} /* string */ 198 | .kw {color: #264D66;} /* keyword */ 199 | .co {color: #888888;} /* comment */ 200 | 201 | .message { color: black; font-weight: bolder;} 202 | .error { color: orange; font-weight: bolder;} 203 | .warning { color: #6A0366; font-weight: bolder;} 204 | 205 | /* Clipboard --------------------------*/ 206 | 207 | .hasCopyButton { 208 | position: relative; 209 | } 210 | 211 | .btn-copy-ex { 212 | position: absolute; 213 | right: 0; 214 | top: 0; 215 | visibility: hidden; 216 | } 217 | 218 | .hasCopyButton:hover button.btn-copy-ex { 219 | visibility: visible; 220 | } 221 | -------------------------------------------------------------------------------- /docs/pkgdown.js: -------------------------------------------------------------------------------- 1 | $(function() { 2 | 3 | $("#sidebar") 4 | .stick_in_parent({offset_top: 40}) 5 | .on('sticky_kit:bottom', function(e) { 6 | $(this).parent().css('position', 'static'); 7 | }) 8 | .on('sticky_kit:unbottom', function(e) { 9 | $(this).parent().css('position', 'relative'); 10 | }); 11 | 12 | $('body').scrollspy({ 13 | target: '#sidebar', 14 | offset: 60 15 | }); 16 | 17 | var cur_path = paths(location.pathname); 18 | $("#navbar ul li a").each(function(index, value) { 19 | if (value.text == "Home") 20 | return; 21 | if (value.getAttribute("href") === "#") 22 | return; 23 | 24 | var path = paths(value.pathname); 25 | if (is_prefix(cur_path, path)) { 26 | // Add class to parent
  • , and enclosing
  • if in dropdown 27 | var menu_anchor = $(value); 28 | menu_anchor.parent().addClass("active"); 29 | menu_anchor.closest("li.dropdown").addClass("active"); 30 | } 31 | }); 32 | }); 33 | 34 | function paths(pathname) { 35 | var pieces = pathname.split("/"); 36 | pieces.shift(); // always starts with / 37 | 38 | var end = pieces[pieces.length - 1]; 39 | if (end === "index.html" || end === "") 40 | pieces.pop(); 41 | return(pieces); 42 | } 43 | 44 | function is_prefix(needle, haystack) { 45 | if (needle.length > haystack.lengh) 46 | return(false); 47 | 48 | // Special case for length-0 haystack, since for loop won't run 49 | if (haystack.length === 0) { 50 | return(needle.length === 0); 51 | } 52 | 53 | for (var i = 0; i < haystack.length; i++) { 54 | if (needle[i] != haystack[i]) 55 | return(false); 56 | } 57 | 58 | return(true); 59 | } 60 | 61 | /* Clipboard --------------------------*/ 62 | 63 | function changeTooltipMessage(element, msg) { 64 | var tooltipOriginalTitle=element.getAttribute('data-original-title'); 65 | element.setAttribute('data-original-title', msg); 66 | $(element).tooltip('show'); 67 | element.setAttribute('data-original-title', tooltipOriginalTitle); 68 | } 69 | 70 | if(Clipboard.isSupported()) { 71 | $(document).ready(function() { 72 | var copyButton = ""; 73 | 74 | $(".examples").addClass("hasCopyButton"); 75 | 76 | // Insert copy buttons: 77 | $(copyButton).prependTo(".hasCopyButton"); 78 | 79 | // Initialize tooltips: 80 | $('.btn-copy-ex').tooltip({container: 'body'}); 81 | 82 | // Initialize clipboard: 83 | var clipboardBtnCopies = new Clipboard('[data-clipboard-copy]', { 84 | text: function(trigger) { 85 | return trigger.parentNode.textContent; 86 | } 87 | }); 88 | 89 | clipboardBtnCopies.on('success', function(e) { 90 | changeTooltipMessage(e.trigger, 'Copied!'); 91 | e.clearSelection(); 92 | }); 93 | 94 | clipboardBtnCopies.on('error', function() { 95 | changeTooltipMessage(e.trigger,'Press Ctrl+C or Command+C to copy'); 96 | }); 97 | }); 98 | } 99 | 100 | -------------------------------------------------------------------------------- /docs/pkgdown.yml: -------------------------------------------------------------------------------- 1 | pandoc: 1.19.2.1 2 | pkgdown: 0.1.0.9000 3 | pkgdown_sha: 7cbafcd6b97e5108ccde816a00d300152534a192 4 | articles: [] 5 | 6 | -------------------------------------------------------------------------------- /docs/reference/csvy.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Import and Export CSV Data With a YAML Metadata Header — csvy • csvy 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 39 | 40 | 41 | 42 | 43 | 44 |
    45 |
    46 | 83 | 84 | 85 |
    86 | 87 |
    88 |
    89 |
    90 |

    Import and Export CSV Data With a YAML Metadata Header

    91 | Source: R/csvy-package.R 92 |
    93 | 94 | 95 |

    CSVY is a file format that combines the simplicity of CSV (comma-separated values) with the metadata of other plain text and binary formats (JSON, XML, Stata, etc.). The CSVY file specification is simple: place a YAML header on top of a regular CSV. The csvy package implements this format using two functions: write_csvy and read_csvy.

    96 | 97 | 98 | 99 | 100 |
    101 | 107 |
    108 | 109 |
    110 |
    111 |

    Developed by Thomas J. Leeper.

    112 |
    113 | 114 |
    115 |

    Site built with pkgdown.

    116 |
    117 | 118 |
    119 |
    120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /docs/reference/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Function reference • csvy 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 36 | 37 | 38 | 39 | 40 | 41 |
    42 |
    43 | 80 | 81 | 82 |
    83 | 84 |
    85 |
    86 |
    87 |

    88 | Reference 89 | version 0.2.1 90 |

    91 |
    92 | 93 |
    94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 107 | 108 | 109 | 110 | 113 | 114 | 115 | 116 | 119 | 120 | 121 | 122 | 125 | 126 | 127 | 128 |
    104 |

    All functions

    105 |

    106 |
    111 |

    csvy

    112 |

    Import and Export CSV Data With a YAML Metadata Header

    117 |

    read_csvy()

    118 |

    Import CSVY data

    123 |

    write_csvy()

    124 |

    Export CSVY data

    129 |
    130 |
    131 | 132 | 138 |
    139 | 140 |
    141 |
    142 |

    Developed by Thomas J. Leeper.

    143 |
    144 | 145 |
    146 |

    Site built with pkgdown.

    147 |
    148 | 149 |
    150 |
    151 | 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /docs/reference/read_csvy.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Import CSVY data — read_csvy • csvy 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 39 | 40 | 41 | 42 | 43 | 44 |
    45 |
    46 | 83 | 84 | 85 |
    86 | 87 |
    88 |
    89 |
    90 |

    Import CSVY data

    91 | Source: R/read_csvy.R 92 |
    93 | 94 | 95 |

    Import CSVY data as a data.frame

    96 | 97 | 98 | 
    read_csvy(file, metadata = NULL, stringsAsFactors = FALSE, ...)
    99 | 100 |

    Arguments

    101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 |
    file

    A character string or R connection specifying a file.

    metadata

    Optionally, a character string specifying a YAML (“.yaml”) or JSON (“.json”) file containing metadata (in lieu of including it in the header of the file).

    stringsAsFactors

    A logical specifying whether to treat character columns as factors. Passed to read.csv or fread depending on the value of method. Ignored for method = 'readr' which never returns factors.

    …

    Additional arguments passed to fread.

    120 | 121 |

    See also

    122 | 123 |

    write_csvy

    124 | 125 | 126 |

    Examples

    127 | 
    read_csvy(system.file("examples", "example3.csvy", package = "csvy"))
    #>   var1 var2 var3
128 | #> 1    A    1  2.0
129 | #> 2    B    3  4.3
    
130 |
    131 |
    132 | 143 |
    144 | 145 |
    146 |
    147 |

    Developed by Thomas J. Leeper.

    148 |
    149 | 150 |
    151 |

    Site built with pkgdown.

    152 |
    153 | 154 |
    155 |
    156 | 157 | 158 | 159 | 160 | -------------------------------------------------------------------------------- /docs/reference/write_csvy.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Export CSVY data — write_csvy • csvy 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 39 | 40 | 41 | 42 | 43 | 44 |
    45 |
    46 | 83 | 84 | 85 |
    86 | 87 |
    88 |
    89 |
    90 |

    Export CSVY data

    91 | Source: R/write_csvy.R 92 |
    93 | 94 | 95 |

    Export data.frame to CSVY

    96 | 97 | 98 | 
    write_csvy(x, file, metadata = NULL, sep = ",", sep2 = ".",
 99 |   comment_header = if (is.null(metadata)) TRUE else FALSE,
100 |   name = as.character(substitute(x)), ...)
    101 | 102 |

    Arguments

    103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 |
    x

    A data.frame.

    file

    A character string or R connection specifying a file.

    metadata

    Optionally, a character string specifying a YAML (“.yaml”) or JSON (“.json”) file to write the metadata (in lieu of including it in the header of the file).

    sep

    A character string specifying a between-field separator. Passed to fwrite.

    sep2

    A character string specifying a within-field separator. Passed to fwrite.

    comment_header

    A logical indicating whether to comment the lines containing the YAML front matter. Default is TRUE.

    name

    A character string specifying a name for the dataset.

    …

    Additional arguments passed to fwrite.

    138 | 139 |

    See also

    140 | 141 |

    write_csvy

    142 | 143 | 144 |

    Examples

    145 | 
    library("datasets")
146 | write_csvy(head(iris))
    #> #---
147 | #> #profile: tabular-data-package
148 | #> #name:
149 | #> #- head
150 | #> #- iris
151 | #> #resources:
152 | #> #- order: 1
153 | #> #  schema:
154 | #> #    fields:
155 | #> #    - name: Sepal.Length
156 | #> #      type: number
157 | #> #    - name: Sepal.Width
158 | #> #      type: number
159 | #> #    - name: Petal.Length
160 | #> #      type: number
161 | #> #    - name: Petal.Width
162 | #> #      type: number
163 | #> #    - name: Species
164 | #> #      type: string
165 | #> #      levels:
166 | #> #      - setosa
167 | #> #      - versicolor
168 | #> #      - virginica
169 | #> #  dialect:
170 | #> #    csvddfVersion: 1.0
171 | #> #    delimiter: ','
172 | #> #    doubleQuote: no
173 | #> #    lineTerminator: \n
174 | #> #    escapeChar: \
175 | #> #    quoteChar: '"'
176 | #> #    skipInitialSpace: yes
177 | #> #    header: yes
178 | #> #    caseSensitiveHeader: yes
179 | #> #--- 
180 | #> Sepal.Length,Sepal.Width,Petal.Length,Petal.Width,Species
181 | #> 5.1,3.5,1.4,0.2,setosa
182 | #> 4.9,3,1.4,0.2,setosa
183 | #> 4.7,3.2,1.3,0.2,setosa
184 | #> 4.6,3.1,1.5,0.2,setosa
185 | #> 5,3.6,1.4,0.2,setosa
186 | #> 5.4,3.9,1.7,0.4,setosa
    
187 | # write yaml w/o comment charaters
188 | write_csvy(head(iris), comment_header = FALSE)
    #> ---
189 | #> profile: tabular-data-package
190 | #> name:
191 | #> - head
192 | #> - iris
193 | #> resources:
194 | #> - order: 1
195 | #>   schema:
196 | #>     fields:
197 | #>     - name: Sepal.Length
198 | #>       type: number
199 | #>     - name: Sepal.Width
200 | #>       type: number
201 | #>     - name: Petal.Length
202 | #>       type: number
203 | #>     - name: Petal.Width
204 | #>       type: number
205 | #>     - name: Species
206 | #>       type: string
207 | #>       levels:
208 | #>       - setosa
209 | #>       - versicolor
210 | #>       - virginica
211 | #>   dialect:
212 | #>     csvddfVersion: 1.0
213 | #>     delimiter: ','
214 | #>     doubleQuote: no
215 | #>     lineTerminator: \n
216 | #>     escapeChar: \
217 | #>     quoteChar: '"'
218 | #>     skipInitialSpace: yes
219 | #>     header: yes
220 | #>     caseSensitiveHeader: yes
221 | #> ---
222 | #> Sepal.Length,Sepal.Width,Petal.Length,Petal.Width,Species
223 | #> 5.1,3.5,1.4,0.2,setosa
224 | #> 4.9,3,1.4,0.2,setosa
225 | #> 4.7,3.2,1.3,0.2,setosa
226 | #> 4.6,3.1,1.5,0.2,setosa
227 | #> 5,3.6,1.4,0.2,setosa
228 | #> 5.4,3.9,1.7,0.4,setosa
    
229 |
    230 |
    231 | 242 |
    243 | 244 | 254 |
    255 | 256 | 257 | 258 | 259 | -------------------------------------------------------------------------------- /inst/CITATION: -------------------------------------------------------------------------------- 1 | citHeader("To cite package 'csvy' in publications use:") 2 | 3 | year <- sub(".*(2[[:digit:]]{3})-.*", "\\1", meta$Date, perl = TRUE) 4 | vers <- paste("R package version", meta$Version) 5 | 6 | citEntry(entry="Manual", 7 | title = "csvy: Import and Export CSV Data with a YAML Metadata Header", 8 | author = personList(as.person("Thomas J. Leeper")), 9 | year = year, 10 | note = vers, 11 | textVersion = 12 | paste("Thomas J. Leeper (", 13 | year, 14 | "). csvy: Import and Export CSV Data With a YAML Metadata Header. ", 15 | vers, ".", sep="")) 16 | -------------------------------------------------------------------------------- /inst/examples/example1.csvy: -------------------------------------------------------------------------------- 1 | --- 2 | profile: tabular-data-resource 3 | name: my-dataset 4 | path: https://raw.githubusercontent.com/csvy/csvy.github.io/master/examples/example.csvy 5 | title: Example file of csvy 6 | description: Show a csvy sample file. 7 | format: csvy 8 | mediatype: text/vnd.yaml 9 | encoding: utf-8 10 | schema: 11 | fields: 12 | - name: var1 13 | type: string 14 | - name: var2 15 | type: integer 16 | - name: var3 17 | type: number 18 | dialect: 19 | csvddfVersion: 1.0 20 | delimiter: "," 21 | doubleQuote: false 22 | lineTerminator: "\r\n" 23 | quoteChar: "\"" 24 | skipInitialSpace: true 25 | header: true 26 | sources: 27 | - title: The csvy specifications 28 | path: http://csvy.org/ 29 | email: '' 30 | licenses: 31 | - name: CC-BY-4.0 32 | title: Creative Commons Attribution 4.0 33 | path: https://creativecommons.org/licenses/by/4.0/ 34 | --- 35 | var1,var2,var3 36 | A,1,2.0 37 | B,3,4.3 38 | -------------------------------------------------------------------------------- /inst/examples/example2.csvy: -------------------------------------------------------------------------------- 1 | #--- 2 | #profile: tabular-data-resource 3 | #name: my-dataset 4 | #path: https://raw.githubusercontent.com/csvy/csvy.github.io/master/examples/example.csvy 5 | #title: Example file of csvy 6 | #description: Show a csvy sample file. 7 | #format: csvy 8 | #mediatype: text/vnd.yaml 9 | #encoding: utf-8 10 | #schema: 11 | # fields: 12 | # - name: var1 13 | # type: string 14 | # - name: var2 15 | # type: integer 16 | # - name: var3 17 | # type: number 18 | #dialect: 19 | # csvddfVersion: 1.0 20 | # delimiter: "," 21 | # doubleQuote: false 22 | # lineTerminator: "\r\n" 23 | # quoteChar: "\"" 24 | # skipInitialSpace: true 25 | # header: true 26 | #sources: 27 | #- title: The csvy specifications 28 | # path: http://csvy.org/ 29 | # email: '' 30 | #licenses: 31 | #- name: CC-BY-4.0 32 | # title: Creative Commons Attribution 4.0 33 | # path: https://creativecommons.org/licenses/by/4.0/ 34 | #--- 35 | var1,var2,var3 36 | A,1,2.0 37 | B,3,4.3 38 | -------------------------------------------------------------------------------- /man/colclass_dict.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/colclass_dict.R 3 | \docType{data} 4 | \name{colclass_dict} 5 | \alias{colclass_dict} 6 | \title{Dictionary of column classes for reading data} 7 | \format{An object of class \code{character} of length 7.} 8 | \usage{ 9 | colclass_dict 10 | } 11 | \description{ 12 | Dictionary of column classes for reading data 13 | } 14 | \keyword{datasets} 15 | -------------------------------------------------------------------------------- /man/csvy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/csvy-package.R 3 | \docType{package} 4 | \name{csvy} 5 | \alias{csvy} 6 | \alias{csvy-package} 7 | \alias{csvy-package} 8 | \title{Import and Export CSV Data With a YAML Metadata Header} 9 | \description{ 10 | CSVY is a file format that combines the simplicity of CSV (comma-separated values) with the metadata of other plain text and binary formats (JSON, XML, Stata, etc.). The \href{http://csvy.org/}{CSVY file specification} is simple: place a YAML header on top of a regular CSV. The csvy package implements this format using two functions: \code{\link{write_csvy}} and \code{\link{read_csvy}}. 11 | } 12 | -------------------------------------------------------------------------------- /man/get_yaml_header.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/get_yaml_header.R 3 | \name{get_yaml_header} 4 | \alias{get_yaml_header} 5 | \title{Retrieve YAML header from file} 6 | \usage{ 7 | get_yaml_header(file, yaml_rxp = "^\\\\#*---[[:space:]]*$", verbose = TRUE) 8 | } 9 | \arguments{ 10 | \item{file}{A character string or R connection specifying a file.} 11 | 12 | \item{yaml_rxp}{Regular expression for parsing YAML header} 13 | 14 | \item{verbose}{Logical. If \code{TRUE}, print warning if no header found.} 15 | } 16 | \value{ 17 | Character vector of lines containing YAML header, or `NULL` if no YAML header found. 18 | } 19 | \description{ 20 | Note that this assumes only one Yaml header, starting on the first line of the file. 21 | } 22 | -------------------------------------------------------------------------------- /man/read_csvy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/read_csvy.R 3 | \name{read_csvy} 4 | \alias{read_csvy} 5 | \title{Import CSVY data} 6 | \usage{ 7 | read_csvy(file, metadata = NULL, stringsAsFactors = FALSE, 8 | detect_metadata = TRUE, ...) 9 | } 10 | \arguments{ 11 | \item{file}{A character string or R connection specifying a file.} 12 | 13 | \item{metadata}{Optionally, a character string specifying a YAML (\dQuote{.yaml}) or JSON (\dQuote{.json}) file containing metadata (in lieu of including it in the header of the file).} 14 | 15 | \item{stringsAsFactors}{A logical specifying whether to treat character columns as factors. Passed to \code{\link[utils]{read.csv}} or \code{\link[data.table]{fread}} depending on the value of \code{method}. Ignored for \code{method = 'readr'} which never returns factors.} 16 | 17 | \item{detect_metadata}{A logical specifying whether to auto-detect a metadata file if none is specified (and if no header is found).} 18 | 19 | \item{\dots}{Additional arguments passed to \code{\link[data.table]{fread}}.} 20 | } 21 | \description{ 22 | Import CSVY data as a data.frame 23 | } 24 | \examples{ 25 | read_csvy(system.file("examples", "example1.csvy", package = "csvy")) 26 | 27 | } 28 | \seealso{ 29 | \code{\link{write_csvy}} 30 | } 31 | -------------------------------------------------------------------------------- /man/read_metadata.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/read_metadata.R 3 | \name{read_metadata} 4 | \alias{read_metadata} 5 | \title{Read metadata} 6 | \usage{ 7 | read_metadata(file) 8 | } 9 | \arguments{ 10 | \item{file}{full path of file from which to read the metadata.} 11 | } 12 | \value{ 13 | the metadata as a list 14 | } 15 | \description{ 16 | Read csvy metadata from an external \code{.yml/.yaml} or \code{.json} file 17 | } 18 | -------------------------------------------------------------------------------- /man/write_csvy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/write_csvy.R 3 | \name{write_csvy} 4 | \alias{write_csvy} 5 | \title{Export CSVY data} 6 | \usage{ 7 | write_csvy(x, file, metadata = NULL, sep = ",", dec = ".", 8 | comment_header = if (is.null(metadata)) TRUE else FALSE, 9 | name = deparse(substitute(x)), metadata_only = FALSE, ...) 10 | } 11 | \arguments{ 12 | \item{x}{A data.frame.} 13 | 14 | \item{file}{A character string or R connection specifying a file.} 15 | 16 | \item{metadata}{Optionally, a character string specifying a YAML (\dQuote{.yaml}) or JSON (\dQuote{.json}) file to write the metadata (in lieu of including it in the header of the file).} 17 | 18 | \item{sep}{A character string specifying a between-field separator. Passed to \code{\link[data.table]{fwrite}}.} 19 | 20 | \item{dec}{A character string specifying a within-field separator. Passed to \code{\link[data.table]{fwrite}}.} 21 | 22 | \item{comment_header}{A logical indicating whether to comment the lines containing the YAML front matter. Default is \code{TRUE}.} 23 | 24 | \item{name}{A character string specifying a name for the dataset.} 25 | 26 | \item{metadata_only}{A logical indicating whether only the metadata should be produced (no CSV component).} 27 | 28 | \item{\dots}{Additional arguments passed to \code{\link[data.table]{fwrite}}.} 29 | } 30 | \description{ 31 | Export data.frame to CSVY 32 | } 33 | \examples{ 34 | library("datasets") 35 | write_csvy(head(iris)) 36 | 37 | # write yaml w/o comment charaters 38 | write_csvy(head(iris), comment_header = FALSE) 39 | 40 | } 41 | \seealso{ 42 | \code{\link{write_csvy}} 43 | } 44 | -------------------------------------------------------------------------------- /man/write_metadata.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/write_metadata.R 3 | \name{write_metadata} 4 | \alias{write_metadata} 5 | \title{Write csvy metadata} 6 | \usage{ 7 | write_metadata(metadata_list = NULL, file = NULL) 8 | } 9 | \arguments{ 10 | \item{metadata_list}{metadata to be stored. Must be valid as per 11 | \code{\link[yaml:as.yaml]{yaml::as.yaml()}} or \code{\link[jsonlite:write_json]{jsonlite::write_json()}} for that particular output 12 | type.} 13 | 14 | \item{file}{full path of file in which to save the metadata.} 15 | } 16 | \value{ 17 | \code{NULL} (invisibly) 18 | } 19 | \description{ 20 | Write csvy metadata to an external \code{.yml/.yaml} or \code{.json} file 21 | } 22 | -------------------------------------------------------------------------------- /tests/test-all.R: -------------------------------------------------------------------------------- 1 | library("testthat") 2 | library("csvy") 3 | test_check("csvy", reporter = "summary") 4 | -------------------------------------------------------------------------------- /tests/testthat/test-metadata.R: -------------------------------------------------------------------------------- 1 | context("CSVY import/export with additional metadata") 2 | require("datasets") 3 | 4 | test_that("Metadata read regardless of comment character in header", { 5 | e1 <- system.file("examples", "example1.csvy", package = "csvy") 6 | e2 <- system.file("examples", "example2.csvy", package = "csvy") 7 | expect_true(identical(get_yaml_header(e1), get_yaml_header(e2))) 8 | }) 9 | 10 | test_that("Metadata supported by read_csvy() and write_csvy()", { 11 | # setup metadata 12 | iris2 <- iris 13 | attr(iris2$Sepal.Length, "label") <- "Sepal Length" 14 | attr(iris2$Sepal.Width, "label") <- "Sepal Width" 15 | attr(iris2$Petal.Length, "label") <- "Petal Length" 16 | attr(iris2$Petal.Width, "label") <- "Petal Width" 17 | 18 | # export 19 | tmp <- tempfile() 20 | write_csvy(iris2, tmp, name = "Edgar Anderson's Iris Data") 21 | suppressWarnings(expect_true(file.exists(tmp), label = "export works with metadata")) 22 | 23 | # import 24 | iris3 <- read_csvy(tmp) 25 | expect_true(attr(iris3, "name") == "Edgar Anderson's Iris Data") 26 | expect_true(attr(iris3$Sepal.Length, "label") == "Sepal Length") 27 | # cleanup 28 | unlink(tmp) 29 | }) 30 | 31 | context("External metadata can be found and loaded") 32 | 33 | test_that("External metadata loads automatically", { 34 | iris2 <- iris 35 | attr(iris2$Sepal.Length, "label") <- "Sepal Length" # to be fixed in PR#21 36 | attr(iris2$Sepal.Width, "label") <- "Sepal Width" # to be fixed in PR#21 37 | attr(iris2$Petal.Length, "label") <- "Petal Length" # to be fixed in PR#21 38 | attr(iris2$Petal.Width, "label") <- "Petal Width" # to be fixed in PR#21 39 | 40 | # export 41 | tmp <- tempfile("iris", fileext = ".csvy") 42 | tmp_metadata <- sub("csvy", "yaml", tmp) 43 | write_csvy(iris2, tmp, metadata = tmp_metadata, name = "Edgar Anderson's Iris Data") 44 | expect_true(file.exists(tmp)) 45 | expect_true(file.exists(tmp_metadata)) 46 | expect_identical(readLines(dir(dirname(tmp), pattern = ".csvy", full.names = TRUE))[1], 47 | "Sepal.Length,Sepal.Width,Petal.Length,Petal.Width,Species") 48 | expect_identical(readLines(dir(dirname(tmp), pattern = ".yaml", full.names = TRUE))[1], 49 | "profile: tabular-data-package") 50 | ## read only csv, auto-detecect metadata 51 | iris3 <- read_csvy(file = tmp) # read metadata automatically 52 | expect_true(attr(iris3, "name") == "Edgar Anderson's Iris Data") 53 | # expect_true(attr(iris3$Sepal.Length, "label") == "Sepal Length") # to be fixed in PR#21 54 | unlink(tmp) 55 | unlink(tmp_metadata) 56 | 57 | }) 58 | 59 | context("read_metadata") 60 | 61 | test_that("empty/bad arguments fail", { 62 | expect_error(read_metadata(), regexp = "argument \"file\" is missing") 63 | expect_error(read_metadata("foo.txt"), regexp = "should be either a \\.json") 64 | }) 65 | 66 | test_that("basic yaml reading works", { 67 | temp_file <- tempfile(fileext = ".yaml") 68 | cat(yaml::as.yaml(list(a = 123.4, foo = "bar")), file = temp_file) 69 | dat <- read_metadata(temp_file) 70 | expect_identical(dat$a, 123.4) 71 | expect_identical(dat$foo, "bar") 72 | unlink(temp_file) 73 | }) 74 | 75 | test_that("basic json reading works", { 76 | temp_file <- tempfile(fileext = ".json") 77 | jsonlite::write_json(list(a = 123.4, foo = "bar"), path = temp_file) 78 | dat <- read_metadata(temp_file) 79 | expect_identical(dat$a, 123.4) 80 | expect_identical(dat$foo, "bar") 81 | unlink(temp_file) 82 | }) 83 | 84 | -------------------------------------------------------------------------------- /tests/testthat/test-read_csvy.R: -------------------------------------------------------------------------------- 1 | context("CSVY import using read_csvy()") 2 | library("datasets") 3 | 4 | test_that("Basic import from CSVY", { 5 | d1 <- read_csvy(system.file("examples", "example1.csvy", package = "csvy")) 6 | expect_true(inherits(d1, "data.frame")) 7 | expect_true(identical(dim(d1), c(2L, 3L))) 8 | 9 | d2 <- read_csvy(system.file("examples", "example2.csvy", package = "csvy")) 10 | expect_true(inherits(d2, "data.frame")) 11 | expect_true(identical(dim(d2), c(2L, 3L))) 12 | }) 13 | 14 | test_that("Import from CSVY with separate yaml header", { 15 | tmp_csvy <- tempfile(fileext = ".csv") 16 | tmp_yaml <- tempfile(fileext = ".yaml") 17 | write_csvy(iris, file = tmp_csvy, metadata = tmp_yaml) 18 | expect_true(inherits(read_csvy(tmp_csvy, metadata = tmp_yaml), "data.frame")) 19 | unlink(tmp_csvy) 20 | unlink(tmp_yaml) 21 | }) 22 | -------------------------------------------------------------------------------- /tests/testthat/test-write_csvy.R: -------------------------------------------------------------------------------- 1 | context("CSVY export using write_csvy()") 2 | require("datasets") 3 | 4 | test_that("Basic export to CSVY", { 5 | tmp <- tempfile() 6 | write_csvy(iris, tmp) 7 | suppressWarnings(expect_true(file.exists(tmp))) 8 | unlink(tmp) 9 | }) 10 | 11 | test_that("Export to CSVY with separate yaml header", { 12 | tmp_csvy <- tempfile(fileext = ".csv") 13 | tmp_yaml <- tempfile(fileext = ".yaml") 14 | tmp_json <- tempfile(fileext = ".json") 15 | 16 | # write to yaml 17 | write_csvy(iris, file = tmp_csvy, metadata = tmp_yaml) 18 | suppressWarnings(expect_true(file.exists(tmp_csvy))) 19 | suppressWarnings(expect_true(file.exists(tmp_yaml))) 20 | unlink(tmp_csvy) 21 | unlink(tmp_yaml) 22 | 23 | # write to json 24 | write_csvy(iris, file = tmp_csvy, metadata = tmp_json) 25 | suppressWarnings(expect_true(file.exists(tmp_csvy))) 26 | suppressWarnings(expect_true(file.exists(tmp_json))) 27 | 28 | unlink(tmp_csvy) 29 | unlink(tmp_json) 30 | }) 31 | -------------------------------------------------------------------------------- /tests/testthat/test-write_metadata.R: -------------------------------------------------------------------------------- 1 | context("write_metadata") 2 | 3 | test_that("empty arguments fail", { 4 | expect_error(write_metadata(), regexp = "must provide.*list") 5 | expect_error(write_metadata(list()), regexp = "metadata \\(filename\\)") 6 | }) 7 | 8 | test_that("wrong filetype fails", { 9 | expect_warning(write_metadata(list(), "file.xls"), regexp = "either a \\.json") 10 | }) 11 | 12 | test_that("basic yaml writing works", { 13 | temp_file <- tempfile(fileext = ".yaml") 14 | write_metadata(list(a = 123.4, foo = "bar"), temp_file) 15 | dat <- yaml::read_yaml(temp_file) 16 | expect_identical(dat$a, 123.4) 17 | expect_identical(dat$foo, "bar") 18 | unlink(temp_file) 19 | }) 20 | 21 | test_that("basic json writing works", { 22 | temp_file <- tempfile(fileext = ".json") 23 | write_metadata(list(a = 123.4, foo = "bar"), temp_file) 24 | dat <- jsonlite::read_json(temp_file, simplifyVector = TRUE) 25 | expect_identical(dat$a, 123.4) 26 | expect_identical(dat$foo, "bar") 27 | unlink(temp_file) 28 | }) 29 | 30 | test_that("only metadata is written", { 31 | temp_dir <- tempdir() 32 | temp_file <- tempfile(tmpdir = temp_dir, fileext = ".json") 33 | write_csvy(mtcars, file = file.path(temp_dir, "mtcars.csvy"), metadata = temp_file, metadata_only = TRUE) 34 | dat <- jsonlite::read_json(temp_file, simplifyVector = TRUE) 35 | expect_identical(dat$name, "mtcars") 36 | expect_identical(unlist(dat$fields$type), rep("number", 11)) 37 | unlink(temp_file) 38 | unlink(file.path(temp_dir, "mtcars.csvy")) 39 | }) 40 | 41 | test_that("roundtrip when metadata is written separately", { 42 | temp_dir <- tempdir() 43 | temp_file <- tempfile(tmpdir = temp_dir, fileext = ".json") 44 | write_csvy(mtcars, file = file.path(temp_dir, "mtcars.csvy"), metadata = temp_file, metadata_only = FALSE) 45 | dat <- jsonlite::read_json(temp_file, simplifyVector = TRUE) 46 | expect_identical(dat$name, "mtcars") 47 | expect_identical(unlist(dat$fields$type), rep("number", 11)) 48 | csvy <- read_csvy(file.path(temp_dir, "mtcars.csvy"), metadata = temp_file) 49 | expect_equivalent(csvy, mtcars) 50 | unlink(temp_file) 51 | }) 52 | --------------------------------------------------------------------------------