├── version.txt ├── datascript.R ├── testthat.R ├── testthat └── test-datascript.R ├── .travis.yml ├── config.yml ├── install-packages.R ├── .gitignore ├── LICENSE ├── README.md └── archive.R /version.txt: -------------------------------------------------------------------------------- 1 | 0.11.2 2 | -------------------------------------------------------------------------------- /datascript.R: -------------------------------------------------------------------------------- 1 | print("Have this script run whatever data cleaning you do") 2 | -------------------------------------------------------------------------------- /testthat.R: -------------------------------------------------------------------------------- 1 | # The reporter options ensure that the build will fail if any of the tests fail 2 | 3 | library(testthat) 4 | 5 | test_dir("testthat", reporter = c("check", "progress")) -------------------------------------------------------------------------------- /testthat/test-datascript.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | context("checks that datascript is working") 3 | 4 | test_that("printed value is correct", { 5 | 6 | expect_output(str(source("../datascript.R")), 7 | "Have this script run whatever data cleaning you do") 8 | 9 | }) 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: r 2 | cache: packages 3 | sudo: false 4 | warnings_are_errors: false 5 | git: 6 | depth: false 7 | install: 8 | - Rscript install-packages.R 9 | 10 | script: 11 | - Rscript testthat.R 12 | - Rscript datascript.R 13 | - Rscript archive.R 14 | 15 | after_success: 16 | 17 | -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # GitHub username or organization and repo name 3 | repo: weecology/livedat 4 | 5 | # Email address and user name that you want to associate with the automated deploys 6 | # Generally you want these to be different from your user account so it is clear 7 | # which commits are automated. 8 | deploy_email: "weecologydeploy@weecology.org" 9 | deploy_username: "Weecology Deploy Bot" 10 | --- 11 | -------------------------------------------------------------------------------- /install-packages.R: -------------------------------------------------------------------------------- 1 | # Install pacman if it isn't already installed 2 | if ("pacman" %in% rownames(installed.packages()) == FALSE) install.packages("pacman") 3 | 4 | # Install packages required for analysis 5 | # Add any packages required for you data cleaning and manipulation to the 6 | # `p_load` function call below 7 | # Do not remove the packages already listed here 8 | # they are important for running the livedat repository 9 | 10 | pacman::p_load(git2r, httr, semver, testthat, yaml) 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # History files 2 | .Rhistory 3 | .Rapp.history 4 | 5 | # Session Data files 6 | .RData 7 | 8 | # Example code in package build process 9 | *-Ex.R 10 | 11 | # Output files from R CMD build 12 | /*.tar.gz 13 | 14 | # Output files from R CMD check 15 | /*.Rcheck/ 16 | 17 | # RStudio files 18 | .Rproj.user/ 19 | 20 | # produced vignettes 21 | vignettes/*.html 22 | vignettes/*.pdf 23 | 24 | # OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 25 | .httr-oauth 26 | 27 | # knitr and R markdown default cache directories 28 | /*_cache/ 29 | /cache/ 30 | 31 | # Temporary files created by R markdown 32 | *.utf8.md 33 | *.knit.md 34 | 35 | #Custom view 36 | .DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Ethan White 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/weecology/livedat.svg?branch=master)](https://travis-ci.org/weecology/livedat) 2 | [![License](http://i.creativecommons.org/p/zero/1.0/88x31.png)](https://raw.githubusercontent.com/weecology/livedat/master/LICENSE) 3 | 4 | # livedat 5 | 6 | This is a **Template Repo** designed to assist in setting up a repository for regularly-updated data 7 | (new data are regularly added and need cleaning and curating). Read [our PLOS Biology paper](https://doi.org/10.1371/journal.pbio.3000125) for more details. 8 | 9 | Instructions for creating an updating data workflow can be found at our companion website: 10 | ## [UpdatingData.org](https://www.updatingdata.org/). 11 | 12 | The basic steps are: 13 | 14 | 1. Clone the repository 15 | 2. Configure the repository for your project 16 | 3. Connect to Zenodo 17 | 4. Connect to Travis 18 | 5. Allow automated updating 19 | 6. Add data code 20 | 7. Add data 21 | 8. Add data checks 22 | 9. Update data 23 | 24 | ## Some Thoughts on Security 25 | 26 | The Personal Access Token from GitHub provides full access to your entire account. It is equivalent to having your username and password. Travis and other CI companies are very serious about security, but there is always some risk when using credentials like this. To mitigate security issues you can set up a separate account purely for deploying your data and only give it access to the repositories that require it. Then use that account's Personal Access Token instead of the one for you primary account. -------------------------------------------------------------------------------- /archive.R: -------------------------------------------------------------------------------- 1 | #' Update the version number 2 | #' 3 | #' By default update the minor version number since most changes are new data. 4 | #' If [major] or [patch] is in the last commit summary increment the matching 5 | #' version instead. 6 | 7 | config <- yaml::yaml.load_file("config.yml") 8 | 9 | repo <- git2r::repository(".") 10 | repo_url <- paste("https://github.com/", config$repo, ".git", sep = "") 11 | git2r::remote_add(repo, name = "deploy", url = repo_url) 12 | cred <- git2r::cred_token("GITHUB_TOKEN") 13 | 14 | # Use a special user name so that it is clear which commits are automated 15 | git2r::config(repo, 16 | user.email = config$deploy_email, 17 | user.name = config$deploy_username) 18 | 19 | # Check the most recent commit for version instructions 20 | last_commit <- git2r::commits(repo)[[1]] 21 | current_ver <- semver::parse_version(readLines("version.txt")) 22 | if (grepl("Merge pull request", last_commit['summary'])){ 23 | last_commit <- git2r::commits(repo)[[2]] 24 | } 25 | 26 | if (grepl("\\[no version bump\\]", last_commit['summary'])) { 27 | new_ver <- current_ver 28 | } else if (grepl("\\[major\\]", last_commit['summary'])) { 29 | new_ver <- semver::increment_version(current_ver, "major", 1L) 30 | } else if (grepl("\\[patch\\]", last_commit['summary'])) { 31 | new_ver <- semver::increment_version(current_ver, "patch", 1L) 32 | } else { 33 | new_ver <- semver::increment_version(current_ver, "minor", 1L) 34 | } 35 | 36 | writeLines(as.character(new_ver), "version.txt") 37 | 38 | # set up the new commit 39 | travis_build <- Sys.getenv("TRAVIS_BUILD_NUMBER") 40 | git2r::checkout(repo, branch = "master") 41 | commit_message <- paste("Update data and trigger archive: Travis Build", 42 | travis_build, 43 | "[skip ci]") 44 | 45 | # detect if this is a Travis CI build triggered by Cron 46 | travis_event <- Sys.getenv("TRAVIS_EVENT_TYPE") 47 | if (travis_event == "cron") { 48 | commit_message <- paste(commit_message, "[cron]") 49 | } 50 | 51 | # write out the new version and add the commit 52 | git2r::add(repo, "*") 53 | git2r::commit(repo, message = commit_message) 54 | 55 | # Create a new release to trigger Zenodo archiving 56 | 57 | github_token = Sys.getenv("GITHUB_TOKEN") 58 | pull_request = Sys.getenv("TRAVIS_PULL_REQUEST") 59 | branch = Sys.getenv("TRAVIS_BRANCH") 60 | 61 | # If the version has been incremented, this is not a pull request, 62 | # and it is the master branch of the repo, then push the updated data, 63 | # create a new tag, push the tag and trigger a release. 64 | if (new_ver > current_ver & branch == 'master' & pull_request == 'false'){ 65 | git2r::push(repo, 66 | name = "deploy", 67 | refspec = "refs/heads/master", 68 | credentials = cred) 69 | git2r::tag(repo, as.character(new_ver), paste("v", new_ver, sep="")) 70 | git2r::push(repo, 71 | name = "deploy", 72 | refspec = paste("refs/tags/", new_ver, sep=""), 73 | credentials = cred) 74 | api_release_url = paste("https://api.github.com/repos/", config$repo, "/releases", sep = "") 75 | httr::POST(url = api_release_url, 76 | httr::content_type_json(), 77 | httr::add_headers(Authorization = paste("token", github_token)), 78 | body = paste('{"tag_name":"', new_ver, '"}', sep='')) 79 | } --------------------------------------------------------------------------------