├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── LICENSE.txt ├── README.md ├── cheat-sheet-neonUtilities.key ├── cheat-sheet-neonUtilities.pdf ├── helper_files ├── added_fields.csv ├── chem_bundles.csv ├── other_bundles.csv ├── readme.md ├── release_2021.csv ├── shared_aquatic.csv ├── shared_flights.csv └── table_types.csv ├── logo.png ├── neonUtilities ├── .Rbuildignore ├── DESCRIPTION ├── LICENSE ├── NAMESPACE ├── NEWS.md ├── R │ ├── assignClasses.R │ ├── byEventSIM.R │ ├── byFileAOP.R │ ├── byTileAOP.R │ ├── chem_bundles.R │ ├── cleanUp.R │ ├── convByteSize.R │ ├── datasetQuery.R │ ├── dateConvert.R │ ├── eddyStampCheck.R │ ├── findDatatables.R │ ├── findTablesByFormat.R │ ├── findTablesUnique.R │ ├── footRaster.R │ ├── getAPI.R │ ├── getAPIHeaders.R │ ├── getAttributes.R │ ├── getAvg.R │ ├── getCitation.R │ ├── getDatatable.R │ ├── getEddyLog.R │ ├── getFileUrls.R │ ├── getIssueLog.R │ ├── getNeonDOI.R │ ├── getPackage.R │ ├── getProductInfo.R │ ├── getProductSensors.R │ ├── getReadmePublicationDate.R │ ├── getRecentPublication.R │ ├── getTaxonTable.R │ ├── getTileUrls.R │ ├── getTimeIndex.R │ ├── getTitle.R │ ├── getVariables.R │ ├── getVariablesEddy.R │ ├── getVarsEddy.R │ ├── getZipUrls.R │ ├── listFilesInZip.R │ ├── listZipfiles.R │ ├── loadByProduct.R │ ├── makePosColumns.R │ ├── other_bundles.R │ ├── queryFiles.R │ ├── quietMessages.R │ ├── readTableNEON.R │ ├── schemaAllStrings.R │ ├── schemaFromVar.R │ ├── shared_aquatic.R │ ├── shared_flights.R │ ├── stackByTable.R │ ├── stackDataFilesParallel.R │ ├── stackEddy.R │ ├── stackFromStore.R │ ├── sysdata.rda │ ├── table_types.R │ ├── timeStampSet.R │ ├── transformFileToGeoCSV.R │ ├── unzipZipfileParallel.R │ ├── zipsByProduct.R │ └── zipsByURI.R ├── README.md ├── data-raw │ └── update_table_types.R ├── data │ ├── chem_bundles.rda │ ├── other_bundles.rda │ ├── shared_aquatic.rda │ ├── shared_flights.rda │ └── table_types.rda ├── inst │ └── extdata │ │ ├── NEON_gp.zip │ │ └── expected │ │ └── gp │ │ ├── issueLog_10109.csv │ │ ├── mga_batchResults.csv │ │ ├── mga_labSummary.csv │ │ ├── mga_soilGroupAbundances.csv │ │ ├── readme_10109.txt │ │ ├── validation_10109.csv │ │ └── variables_10109.csv ├── man │ ├── assignClasses.Rd │ ├── byEventSIM.Rd │ ├── byFileAOP.Rd │ ├── byTileAOP.Rd │ ├── chem_bundles.Rd │ ├── cleanUp.Rd │ ├── convByteSize.Rd │ ├── datasetQuery.Rd │ ├── dateConvert.Rd │ ├── eddyStampCheck.Rd │ ├── findDatatables.Rd │ ├── findTablesByFormat.Rd │ ├── findTablesUnique.Rd │ ├── footRaster.Rd │ ├── getAPI.Rd │ ├── getAPIHeaders.Rd │ ├── getAttributes.Rd │ ├── getAvg.Rd │ ├── getCitation.Rd │ ├── getDatatable.Rd │ ├── getEddyLog.Rd │ ├── getFileUrls.Rd │ ├── getIssueLog.Rd │ ├── getNeonDOI.Rd │ ├── getPackage.Rd │ ├── getProductInfo.Rd │ ├── getProductSensors.Rd │ ├── getReadmePublicationDate.Rd │ ├── getRecentPublication.Rd │ ├── getTaxonTable.Rd │ ├── getTileUrls.Rd │ ├── getTimeIndex.Rd │ ├── getTitle.Rd │ ├── getVariables.Rd │ ├── getVariablesEddy.Rd │ ├── getVarsEddy.Rd │ ├── getZipUrls.Rd │ ├── listFilesInZip.Rd │ ├── listZipfiles.Rd │ ├── loadByProduct.Rd │ ├── makePosColumns.Rd │ ├── other_bundles.Rd │ ├── queryFiles.Rd │ ├── quietMessages.Rd │ ├── readTableNEON.Rd │ ├── schemaAllStrings.Rd │ ├── schemaFromVar.Rd │ ├── shared_aquatic.Rd │ ├── shared_flights.Rd │ ├── stackByTable.Rd │ ├── stackDataFilesParallel.Rd │ ├── stackEddy.Rd │ ├── stackFromStore.Rd │ ├── table_types.Rd │ ├── timeStampSet.Rd │ ├── transformFileToGeoCSV.Rd │ ├── unzipZipfileParallel.Rd │ ├── zipsByProduct.Rd │ └── zipsByURI.Rd ├── neonUtilities.Rproj └── tests │ ├── testthat.R │ └── testthat │ ├── test_byTileAOP.R │ └── test_stackByTable.R └── release_archive ├── neonUtilities_1.0.0.tar.gz ├── neonUtilities_1.0.1.tar.gz ├── neonUtilities_1.2.0.tar.gz ├── neonUtilities_1.2.1.tar.gz ├── neonUtilities_1.2.2.tar.gz ├── neonUtilities_1.3.0.tar.gz ├── neonUtilities_1.3.1.tar.gz ├── neonUtilities_1.3.2.tar.gz ├── neonUtilities_1.3.3.tar.gz ├── neonUtilities_1.3.4.tar.gz ├── neonUtilities_1.3.5.tar.gz ├── neonUtilities_1.3.6.tar.gz ├── neonUtilities_1.3.7.tar.gz ├── neonUtilities_1.3.8.tar.gz ├── neonUtilities_1.3.9.tar.gz ├── neonUtilities_2.0.0.tar.gz ├── neonUtilities_2.0.1.tar.gz ├── neonUtilities_2.1.0.tar.gz ├── neonUtilities_2.1.1.tar.gz ├── neonUtilities_2.1.2.tar.gz ├── neonUtilities_2.1.3.tar.gz ├── neonUtilities_2.1.4.tar.gz ├── neonUtilities_2.2.0.tar.gz ├── neonUtilities_2.2.1.tar.gz ├── neonUtilities_2.3.0.tar.gz ├── neonUtilities_2.4.0.tar.gz ├── neonUtilities_2.4.1.tar.gz ├── neonUtilities_2.4.2.tar.gz └── neonUtilities_2.4.3.tar.gz /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Alert us to a problem 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Function** 11 | Which of the functions in the neonUtilities package is this bug found in? 12 | 13 | **Describe the bug** 14 | A clear and concise description of what the bug is. 15 | 16 | **To Reproduce** 17 | What function inputs revealed the bug? 18 | 19 | **Expected behavior** 20 | A clear and concise description of what you expected to happen. 21 | 22 | **System (please complete the following information):** 23 | - OS: [e.g. iOS] 24 | - OS Version: [e.g. 22] 25 | - R Version: [e.g. 3.5.1] 26 | 27 | **Additional context** 28 | Add any other context about the problem here. 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest new functionality 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | .DS_Store 6 | *.Rproj 7 | cheat-sheet-template.pptx 8 | neonUtilities-cheat-sheet.pptx -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | NEON-utilities 2 | ================ 3 | 4 | 5 | 6 | 7 | NEON-utilities is a repository hosting packages and code for basic data handling for NEON data. It currently contains: 8 | 9 | 1. `neonUtilities`: an R package for discovering, downloading, and working with NEON data files. This builds upon the deprecated pacakge, `neonDataStackR`. 10 | 2. `neon-data-stacker-python`: a Python script for combining the monthly files delivered by the NEON data portal into a single file for each table type. This script is a prototype version and not for general use. It is being **deprecated** in favor of using the `neonUtilities` R package within Python. Please see the Notebook in the neon-data-stacker-python directory or use the NEON Science Using NEON Utilities in Python tutorial to learn more. 11 | 12 | 13 | Requests 14 | -------- 15 | 16 | Bug reports and feature requests should be posted to the Issues page in this repository. To see current priorities for package improvement, see the neonUtilities project on the NEON public code projects board. 17 | 18 | 19 | 20 | Usage 21 | ----- 22 | 23 | Specific instructions for using each code/package are provided in the README in each code/package subdirectory. 24 | 25 | If you encounter problems with code in this repository, feel free to post an issue. 26 | 27 | 28 | Credits & Acknowledgements 29 | -------------------------- 30 | 31 | 32 | The National Ecological Observatory Network is a project solely funded by the National Science Foundation and managed under cooperative agreement by Battelle. Any opinions, findings, and conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the National Science Foundation. 33 | 34 | 35 | License 36 | ------- 37 | GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 38 | 39 | 40 | Disclaimer 41 | ---------- 42 | *Information and documents contained within this repository are available as-is. Codes or documents, or their use, may not be supported or maintained under any program or service and may not be compatible with data currently available from the NEON Data Portal.* 43 | 44 | -------------------------------------------------------------------------------- /cheat-sheet-neonUtilities.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/cheat-sheet-neonUtilities.key -------------------------------------------------------------------------------- /cheat-sheet-neonUtilities.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/cheat-sheet-neonUtilities.pdf -------------------------------------------------------------------------------- /helper_files/added_fields.csv: -------------------------------------------------------------------------------- 1 | fieldName,description,dataType,units,downloadPkg,pubFormat,primaryKey,categoricalCodeName 2 | domainID,Unique identifier of the NEON domain,string,,appended by stackByTable,,N,"" 3 | siteID,NEON site code,string,,appended by stackByTable,,N,"" 4 | horizontalPosition,Index of horizontal location at a NEON site,string,,appended by stackByTable,,N,"" 5 | verticalPosition,Index of vertical location at a NEON site,string,,appended by stackByTable,,N,"" 6 | publicationDate,Date of data publication on the NEON data portal,dateTime,,appended by stackByTable,,N,"" 7 | release,Identifier for data release,string,,appended by stackByTable,,N,"" 8 | -------------------------------------------------------------------------------- /helper_files/chem_bundles.csv: -------------------------------------------------------------------------------- 1 | product,homeProduct 2 | DP1.10102.001,depends 3 | DP1.10099.001,depends 4 | DP1.10053.001,DP1.10026.001 5 | DP1.10031.001,DP1.10033.001 6 | DP1.10101.001,DP1.10033.001 7 | DP1.10080.001,DP1.10086.001 8 | DP1.10078.001,DP1.10086.001 9 | DP1.10100.001,DP1.10086.001 10 | DP1.10008.001,DP1.10047.001 11 | DP1.00097.001,DP1.00096.001 12 | -------------------------------------------------------------------------------- /helper_files/other_bundles.csv: -------------------------------------------------------------------------------- 1 | product,homeProduct 2 | DP1.10045.001,DP1.10098.001 3 | DP1.20197.001,DP1.20194.001 4 | -------------------------------------------------------------------------------- /helper_files/readme.md: -------------------------------------------------------------------------------- 1 | ## Helper files for neonUtilities functionality. Additional documentation in R package manual. 2 | 3 | * added\_fields 4 | 5 | This table contains variables file metadata for the additional fields that can be added to tabular data during stacking by stackByTable(). The relevant fields are appended to the variables file as needed during the stacking process. 6 | 7 | * chem\_bundles 8 | * other\_bundles 9 | * release\_2021 10 | 11 | Several data products are "bundled" in that, e.g., litter chemical properties can only be downloaded as part of the litterfall data product. The chem\_bundles and other\_bundles tables contain the mapping, so that the download function can redirect to the bundled product if a non-independent product is requested. 12 | 13 | Some of these bundles were created after the 2021 release. The non-independent data tables are no longer accessible via the database query that populates the table\_types table, so the release\_2021 table retains them and is appended to table\_types every time table\_types is regenerated. This enables neonUtilities to continue stacking data from RELEASE-2021 correctly. 14 | 15 | * shared\_aquatic 16 | * shared\_flights 17 | 18 | Some sites are paired for some purposes. For aquatic sites that don't have a meteorological station, shared_aquatic provides the mapping to the nearby terrestrial site to be used for specific data products. The download functions redirect to the nearby terrestrial site if the aquatic site and one of the specified products are requested. 19 | 20 | Some flight boxes cover more than one site. If AOP data are requested for a site that is covered by another site's flight box, shared\_flights provides the mapping and redirects the download to the designated site. 21 | 22 | * table\_types 23 | 24 | There are a few different options for data publication from the NEON pipeline, and they have different implications for data stacking. Data that are published based on the date and site of record in the data (site-date) are straightforward, all available data are stacked. For site-all data tables, all available data are published in every data package for a given site, regardless of date. This is done when the data are not specific to a date, for example the trap establishment data for litter traps. In this case data stacking is performed on the most recently published data for each site. Data that are specific to a lab, such as method detection limits, are published in every data package that contains relevant data from the lab in question. In this case data stacking is performed on the most recently published data for each lab. The table\_types table contains an index of all NEON tabular data tables and their types. If a table is not found on this list, the findTablesByFormat() function attempts to infer the type based on the format of the file name, but the table_types table is the definitive source. 25 | -------------------------------------------------------------------------------- /helper_files/release_2021.csv: -------------------------------------------------------------------------------- 1 | productID,tableName,tableDesc,tableType,tableTMI 2 | DP1.10045.001,vst_perplotperyear,"Per plot sampling metadata, including presence/absence of each growthForm",site-date,0 3 | DP1.10045.001,nst_perindividual,"Field measurements of individual non-herbaceous perennial plants (e.g. cacti, ferns)",site-date,0 4 | DP1.20197.001,asc_fieldDataPoint,Point level sediment collection field data,site-date,0 5 | DP1.20197.001,asc_fieldDataZone,Zone level sediment collection field data,site-date,0 6 | DP1.20197.001,asc_fieldDataStation,Station level sediment collection field data,site-date,0 7 | DP1.20197.001,asp_externalLabData,Sediment physical properties external lab data,site-date,0 8 | DP1.20197.001,asc_externalLabSummary,Sediment chemistry external lab long-term summary data,lab-current,0 9 | -------------------------------------------------------------------------------- /helper_files/shared_aquatic.csv: -------------------------------------------------------------------------------- 1 | site,towerSite,product 2 | HOPB,HARV,DP1.00006.001 3 | HOPB,HARV,DP1.00038.001 4 | HOPB,HARV,DP1.00013.001 5 | POSE,SCBI,DP1.00006.001 6 | POSE,SCBI,DP1.00038.001 7 | POSE,SCBI,DP1.00013.001 8 | LEWI,BLAN,DP1.00006.001 9 | LEWI,BLAN,DP1.00038.001 10 | LEWI,BLAN,DP1.00013.001 11 | BARC,OSBS,DP1.00006.001 12 | BARC,OSBS,DP1.00038.001 13 | BARC,OSBS,DP1.00013.001 14 | SUGG,OSBS,DP1.00006.001 15 | SUGG,OSBS,DP1.00038.001 16 | SUGG,OSBS,DP1.00013.001 17 | FLNT,JERC,DP1.00006.001 18 | FLNT,JERC,DP1.00038.001 19 | FLNT,JERC,DP1.00013.001 20 | CRAM,UNDE,DP1.00006.001 21 | CRAM,UNDE,DP1.00038.001 22 | CRAM,UNDE,DP1.00013.001 23 | LIRO,UNDE,DP1.00006.001 24 | LIRO,UNDE,DP1.00038.001 25 | LIRO,UNDE,DP1.00013.001 26 | KING,KONZ,DP1.00006.001 27 | KING,KONZ,DP1.00038.001 28 | KING,KONZ,DP1.00013.001 29 | WALK,ORNL,DP1.00006.001 30 | WALK,ORNL,DP1.00038.001 31 | WALK,ORNL,DP1.00013.001 32 | LECO,GRSM,DP1.00006.001 33 | LECO,GRSM,DP1.00038.001 34 | LECO,GRSM,DP1.00013.001 35 | MAYF,TALL,DP1.00006.001 36 | MAYF,TALL,DP1.00038.001 37 | MAYF,TALL,DP1.00013.001 38 | BLWA,DELA,DP1.00004.001 39 | BLWA,DELA,DP1.00001.001 40 | BLWA,DELA,DP1.00024.001 41 | BLWA,DELA,DP1.00098.001 42 | BLWA,DELA,DP1.00002.001 43 | BLWA,DELA,DP1.00023.001 44 | BLWA,DELA,DP1.00006.001 45 | BLWA,DELA,DP1.00038.001 46 | BLWA,DELA,DP1.00013.001 47 | TOMB,LENO,DP1.00004.001 48 | TOMB,LENO,DP1.00001.001 49 | TOMB,LENO,DP1.00024.001 50 | TOMB,LENO,DP1.00098.001 51 | TOMB,LENO,DP1.00002.001 52 | TOMB,LENO,DP1.00023.001 53 | TOMB,LENO,DP1.00006.001 54 | TOMB,LENO,DP1.00038.001 55 | TOMB,LENO,DP1.00013.001 56 | PRPO,WOOD,DP1.00006.001 57 | PRPO,WOOD,DP1.00038.001 58 | PRPO,WOOD,DP1.00013.001 59 | PRLA,DCFS,DP1.00006.001 60 | PRLA,DCFS,DP1.00038.001 61 | PRLA,DCFS,DP1.00013.001 62 | BLDE,YELL,DP1.00006.001 63 | BLDE,YELL,DP1.00038.001 64 | BLDE,YELL,DP1.00013.001 65 | COMO,NIWO,DP1.00006.001 66 | COMO,NIWO,DP1.00038.001 67 | COMO,NIWO,DP1.00013.001 68 | MART,WREF,DP1.00006.001 69 | MART,WREF,DP1.00038.001 70 | MART,WREF,DP1.00013.001 71 | TECR,TEAK,DP1.00006.001 72 | TECR,TEAK,DP1.00038.001 73 | TECR,TEAK,DP1.00013.001 74 | OKSR,TOOL,DP1.00006.001 75 | OKSR,TOOL,DP1.00038.001 76 | OKSR,TOOL,DP1.00013.001 77 | CARI,BONA,DP1.00006.001 78 | CARI,BONA,DP1.00038.001 79 | CARI,BONA,DP1.00013.001 80 | -------------------------------------------------------------------------------- /helper_files/shared_flights.csv: -------------------------------------------------------------------------------- 1 | site,flightSite 2 | TREE,STEI 3 | CHEQ,STEI 4 | KONA,KONZ 5 | BARC,OSBS 6 | SUGG,OSBS 7 | COMO,NIWO 8 | MAYF,TALL 9 | FLNT,JERC 10 | TOMB,LENO 11 | BLWA,DELA 12 | POSE,SCBI 13 | CRAM,UNDE 14 | TOOK,TOOL 15 | CARI,BONA 16 | MART,WREF 17 | TECR,TEAK 18 | BIGC,SOAP 19 | BLDE,YELL 20 | PRLA,WOOD 21 | PRPO,WOOD 22 | KING,KONZ 23 | LEWI,BLAN 24 | DCFS,WOOD 25 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/logo.png -------------------------------------------------------------------------------- /neonUtilities/.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | ^data-raw$ 4 | ^LICENSE$ 5 | -------------------------------------------------------------------------------- /neonUtilities/DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: neonUtilities 2 | Version: 3.0.0.9000 3 | Date: 2025-02-04 4 | Title: Utilities for Working with NEON Data 5 | Authors@R: c(person("Claire","Lunch",email="clunch@battelleecology.org",role=c("aut","cre","ctb")), person("Christine","Laney",role=c("aut","ctb")), person("Nathan","Mietkiewicz",role=c("aut","ctb")), person("Eric","Sokol",role=c("aut","ctb")), person("Kaelin","Cawley",role=c("aut","ctb")), person("NEON (National Ecological Observatory Network)", role="aut")) 6 | Description: NEON data packages can be accessed through the NEON Data Portal 7 | or through the NEON Data API (see for documentation). Data delivered from 8 | the Data Portal are provided as monthly zip files packaged within a parent zip file, while individual files 9 | can be accessed from the API. This package provides tools that aid in discovering, downloading, and reformatting 10 | data prior to use in analyses. This includes downloading data via the API, merging data tables by type, and 11 | converting formats. For more information, see the readme file at . 12 | Depends: R (>= 3.6) 13 | Imports: httr, jsonlite, downloader, data.table, utils, R.utils, stats, tidyr, stringr, pbapply, parallel, curl, arrow, rlang 14 | Suggests: rhdf5, terra, testthat, fasttime, dplyr 15 | License: AGPL-3 16 | URL: https://github.com/NEONScience/NEON-utilities 17 | BugReports: https://github.com/NEONScience/NEON-utilities/issues 18 | Encoding: UTF-8 19 | LazyData: true 20 | RoxygenNote: 7.3.2 21 | -------------------------------------------------------------------------------- /neonUtilities/NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | export(byEventSIM) 4 | export(byFileAOP) 5 | export(byTileAOP) 6 | export(convByteSize) 7 | export(datasetQuery) 8 | export(footRaster) 9 | export(getAvg) 10 | export(getCitation) 11 | export(getDatatable) 12 | export(getIssueLog) 13 | export(getNeonDOI) 14 | export(getPackage) 15 | export(getProductInfo) 16 | export(getProductSensors) 17 | export(getTaxonTable) 18 | export(getTimeIndex) 19 | export(getVarsEddy) 20 | export(loadByProduct) 21 | export(queryFiles) 22 | export(readTableNEON) 23 | export(schemaFromVar) 24 | export(stackByTable) 25 | export(stackEddy) 26 | export(stackFromStore) 27 | export(transformFileToGeoCSV) 28 | export(zipsByProduct) 29 | export(zipsByURI) 30 | import(httr) 31 | import(jsonlite) 32 | importFrom(httr,GET) 33 | importFrom(jsonlite,fromJSON) 34 | importFrom(rlang,.data) 35 | -------------------------------------------------------------------------------- /neonUtilities/NEWS.md: -------------------------------------------------------------------------------- 1 | # version 2.4.3 2 | 3 | Released 2024-12-04 4 | 5 | ## Changes 6 | 7 | * stackByTable() and loadByProduct() updated to handle new files associated with Bathymetric and morphological maps (DP4.00132.001) and the three revised microbe community data products (DP1.10081.002, DP1.20086.002, and DP1.20141.002) 8 | 9 | 10 | # version 2.4.2 11 | 12 | Released 2024-04-30 13 | 14 | ## Bug fixes 15 | 16 | * stackEddy() had been erroring out any time it encountered a file that was incompatible with the input criteria (e.g., a variable requested in the var= parameter was not present in an h5 file). Now it skips the file and moves on to the next. 17 | * Download functions (zipsByProduct(), loadByProduct(), byFileAOP(), byTileAOP()) include a re-attempt routine if the first download attempt fails. include.provisional= was not handled correctly in the re-attempt code and resulted in the download erroring out. This is now fixed. 18 | * Downloads by averaging interval (timeIndex=) in zipsByProduct() and loadByProduct() were not retrieving science review flag files. This is now fixed. 19 | 20 | 21 | # version 2.4.1 22 | 23 | Released 2024-01-09 24 | 25 | ## Enhancements 26 | 27 | * New option to run stackEddy() only on already-downloaded data, without pinging the NEON API. Using this option blocks retrieval of the issue log and citation files; its use is not recommended unless you need to run code offline or otherwise need to avoid API calls. Toggle this option using the input `runLocal=`, which defaults to FALSE. 28 | 29 | ## Bug fixes 30 | 31 | * Data tables in Breeding landbird point counts (DP1.10003.001), Bathymetric and morphological maps (DP4.00132.001), and Stream morphology map (DP4.00131.001) have changed publication paradigm over time (e.g., from publication only in the site and month when collection occurred, to publication in all months with data for the relevant site). stackByTable() and loadByProduct() have been updated to accommodate both current data and past releases. 32 | * Surface-atmosphere exchange (Eddy covariance bundle, DP4.00200.001) metadata have updated the formatting of UTM zone reporting; footRaster() has been updated to handle both current and past formatting. 33 | 34 | 35 | # version 2.4.0 36 | 37 | Released 2023-10-17 38 | 39 | ## Changes 40 | 41 | * Data download functions now default to not returning provisional data. This applies to zipsByProduct(), loadByProduct(), byFileAOP(), and byTileAOP(). To download provisional data, add the new parameter `include.provisional=TRUE` to the function call. 42 | * Data stacking has been updated to handle the new formatting of the Reaeration (DP1.20190.001) and Salt-based discharge (DP1.20193.001) data products. The conductivity data tables in these two products have been converted to separate files per sampling event. This update enables stackByTable() and loadByProduct() to work with both the previous and current formats. 43 | 44 | 45 | # version 2.3.0 46 | 47 | Released 2023-07-05 48 | 49 | ## Enhancements 50 | 51 | * Data download and stacking functions now include recommended citation(s) in BibTeX format. Applies to stackByTable(), loadByProduct(), stackEddy(), byFileAOP(), and byTileAOP(). Note that citation retrieval will generally work on data downloaded from the NEON data portal or using neonUtilities functions, but if you use alternative download methods and/or modify the contents of downloaded data packages, the stacking functions may not be able to identify data release information and may not provide citations. 52 | * New parameter metadata= in stackEddy() to retrieve site, code, and CO2 validation data from H5 file attributes. When used with basic package data, the attributes are valid for the first day of each month accessed. To get attributes for each day of data, use with the expanded package. 53 | * Performance improvements to stackEddy(), will mostly be noticeable when stacking large amounts of data. 54 | * Conversion of dates to POSIXct in stacking functions can optionally use the fasttime package, toggle on and off with useFasttime=. Applies to stackByTable(), readTableNEON(), loadByProduct(), and stackEddy(). Default is not to use fasttime, since it can introduce small inconsistencies in time stamps at the millisecond level. But note that for most NEON data products, precision at the millisecond level is unlikely to be scientifically meaningful. 55 | 56 | ## Bug fixes 57 | 58 | * When no data are available for a particular day and sensor in the eddy covariance system, a single empty for the day is reported in the H5 file. In some cases, stackEddy() was mishandling these records and creating duplicate records. This is now resolved. 59 | 60 | ## Other changes 61 | 62 | * byTileAOP() and footRaster() now use the terra package for spatial data handling, replacing the raster package, which will be obsolete as of Oct 2023. 63 | 64 | 65 | ----------------------- 66 | 67 | # 2023-06-30: created NEWS file 68 | -------------------------------------------------------------------------------- /neonUtilities/R/assignClasses.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Assign correct column classes 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Use the variables file to assign classes to each column in each data file 9 | #' 10 | #' @keywords internal 11 | #' @param dt A data frame 12 | #' @param inVars The variables expected in the df 13 | #' @return A data frame with corrected column classes 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # Changelog and author contributions / copyrights 19 | # 2017-09-28 (Christine Laney): Created original function 20 | # 2018-04-13 (Christine Laney): stackDataFiles now generates a data.table rather than a data.frame; 21 | # convert back to data.frame for checking and updating column types, then convert back to data.table. 22 | ############################################################################################## 23 | 24 | assignClasses <- function(dt, inVars){ 25 | df <- as.data.frame(dt) 26 | for(i in 1:ncol(df)){ 27 | if(names(df)[i] %in% inVars$fieldName){ 28 | type <- inVars$colClass[which(inVars$fieldName==names(df)[i])[1]] 29 | if(type == "numeric") 30 | df[,i] <- as.numeric(df[,i]) 31 | if(type == "character") 32 | df[,i] <- as.character(df[,i]) 33 | } 34 | } 35 | dt <- data.table::as.data.table(df) 36 | return(dt) 37 | } 38 | -------------------------------------------------------------------------------- /neonUtilities/R/byEventSIM.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get site management data by event type. 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Query site management data to return records matching a specific eventType. 9 | #' 10 | #' @param eventType The value of eventType to search for. Can be multiple values. See categoricalCodes file for DP1.10111.001 for possible values. 11 | #' @param site Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all. 12 | #' @param startdate Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA. 13 | #' @param enddate Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA. 14 | #' @param release The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'. 15 | #' @param include.provisional T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data. 16 | #' @param token User specific API token (generated within data.neonscience.org user accounts) 17 | 18 | #' @return A data frame of sim_eventData data, matching the query criteria. Note that metadata are not included in the data returned via this function. 19 | 20 | #' @examples 21 | #' \dontrun{ 22 | #' # Search for fires across all NEON event data 23 | #' sim.fires <- byEventSIM(eventType="fire") 24 | #' 25 | #' # Search for grazing events at several sites 26 | #' sim.graz <- byEventSIM(eventType="grazing", site=c("CPER","KONA","MOAB","STER","LAJA")) 27 | #' } 28 | #' 29 | #' @importFrom rlang .data 30 | 31 | #' @references 32 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 33 | 34 | #' @export 35 | 36 | # changelog and author contributions / copyrights 37 | # Claire Lunch (2025-03-11) 38 | # original creation 39 | ############################################################################################## 40 | 41 | byEventSIM <- function(eventType, 42 | site="all", 43 | startdate=NA, 44 | enddate=NA, 45 | release="current", 46 | include.provisional=FALSE, 47 | token=NA_character_) { 48 | 49 | # check for dplyr 50 | if(!requireNamespace("dplyr", quietly=T)) { 51 | stop("The dplyr package is required to use this function. Install and re-try.") 52 | } 53 | 54 | urlset <- queryFiles(dpID="DP1.10111.001", site=site, 55 | package='basic', startdate=startdate, 56 | enddate=enddate, release=release, 57 | tabl='sim_eventData', metadata=TRUE, 58 | include.provisional=include.provisional, 59 | token=token) 60 | 61 | # subset to only the table to query 62 | edlst <- base::grep("sim_eventData", urlset[[1]], value=TRUE) 63 | 64 | # get variables file and translate to schema 65 | varfl <- urlset[[2]] 66 | vschema <- schemaFromVar(varfl, tab="sim_eventData", package="basic") 67 | 68 | # open dataset and query for event type 69 | ds <- arrow::open_csv_dataset(sources=edlst, schema=vschema, skip=1) 70 | 71 | evFilter <- eventType 72 | evds <- dplyr::filter(.data=ds, .data$eventType==evFilter) 73 | events <- data.frame(dplyr::collect(evds)) 74 | 75 | return(events) 76 | 77 | } 78 | -------------------------------------------------------------------------------- /neonUtilities/R/chem_bundles.R: -------------------------------------------------------------------------------- 1 | #' Bundled chemistry data product information 2 | #' 3 | #' A dataset containing NEON data product codes of terrestrial chemistry data products and the "home" data products they are bundled with. 4 | #' 5 | #' @format A data frame with 2 variables: 6 | #' \describe{ 7 | #' \item{product}{Data product ID of a terrestrial chemistry product} 8 | #' \item{homeProduct}{Data product ID of the corresponding home data product} 9 | #' } 10 | #' @source NEON data product bundles 11 | "chem_bundles" 12 | -------------------------------------------------------------------------------- /neonUtilities/R/cleanUp.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Clean up folder after stacking 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Remove unzipped monthly data folders 9 | #' 10 | #' @keywords internal 11 | #' @param folder The file path to the folder that needs to be cleaned up (the root directory of the data package) 12 | #' @param orig The list of files that were present in the folder before unzipping and stacking 13 | #' @return Only the folders created during unzip will be deleted. All custom folders/files and the stackedFiles output folder will be retained. 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # Changelog and author contributions / copyrights 19 | # Christine Laney (2018-01-10) 20 | # Nathan Mietkiewicz (2020-02-12) 21 | ############################################################################################## 22 | cleanUp <- function(folder, orig) { 23 | 24 | currentFileList <- paste(folder, orig, sep="/") 25 | 26 | if(length(currentFileList) > 0) { 27 | unlink(currentFileList, recursive = TRUE) 28 | } 29 | 30 | writeLines("All unzipped monthly data folders have been removed.") 31 | } 32 | -------------------------------------------------------------------------------- /neonUtilities/R/convByteSize.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Convert a number of bytes into megabytes or gigabytes 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' For any number of bytes, convert to a number of MB or GB 9 | #' 10 | #' @keywords internal 11 | #' @param objSize The size in bytes 12 | #' @return The size of the file in megabytes or gigabytes 13 | #' 14 | #' @references 15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 16 | 17 | #' @export 18 | 19 | # changelog and author contributions / copyrights 20 | # Claire Lunch (2020-09-24) 21 | ############################################################################################## 22 | 23 | convByteSize <- function(objSize){ 24 | mbs <- objSize/1e6 25 | if(mbs < 1000) { 26 | return(paste(mbs, " MB", sep="")) 27 | } else { 28 | gbs <- objSize/1e9 29 | return(paste(gbs, " GB", sep="")) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /neonUtilities/R/dateConvert.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Convert date stamps from character 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Attempt to convert date stamps from character, iterating through known NEON date formats 9 | #' 10 | #' @keywords internal 11 | #' @param dates A vector of date values in character format [character] 12 | #' @param useFasttime Should the fasttime package be used for date conversion? Defaults to false. [logical] 13 | #' @return A POSIXct vector, if possible; if conversion was unsuccessful, the original vector is returned 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # Changelog and author contributions / copyrights 19 | # Claire Lunch (2019-11-08) 20 | # Claire Lunch 2023-06-30: Add fasttime option 21 | ############################################################################################## 22 | 23 | dateConvert <- function(dates, useFasttime=FALSE){ 24 | 25 | if(useFasttime) { 26 | d <- try(fasttime::fastPOSIXct(dates, tz='GMT'), silent=T) 27 | if(any(c(class(d)=='try-error', all(is.na(d))))) { 28 | d <- dates 29 | } 30 | } else { 31 | d <- try(as.POSIXct(dates, format='%Y-%m-%dT%H:%M:%S', tz='GMT'), silent=T) 32 | if(any(c(class(d)=='try-error', all(is.na(d))))) { 33 | d <- try(as.POSIXct(dates, format='%Y-%m-%dT%H:%M', tz='GMT'), silent=T) 34 | } 35 | if(any(c(class(d)=='try-error', all(is.na(d))))) { 36 | d <- try(as.POSIXct(dates, format='%Y-%m-%dT%H', tz='GMT'), silent=T) 37 | } 38 | if(any(c(class(d)=='try-error', all(is.na(d))))) { 39 | d <- try(as.POSIXct(dates, format='%Y-%m-%d', tz='GMT'), silent=T) 40 | } 41 | if(any(c(class(d)=='try-error', all(is.na(d))))) { 42 | d <- dates 43 | } 44 | } 45 | return(d) 46 | } 47 | 48 | -------------------------------------------------------------------------------- /neonUtilities/R/eddyStampCheck.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Convert date stamps from character and check for only one record in a day 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Convert SAE time stamps to POSIX and check for missing data 9 | #' 10 | #' @keywords internal 11 | #' @param tab A table of SAE data 12 | #' @param useFasttime Should the fasttime package be used to convert time stamps? 13 | #' @return The same table of SAE data, with time stamps converted and empty records representing a single day (filler records inserted during processing) removed. 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # Changelog and author contributions / copyrights 19 | # Claire Lunch (2019-11-08) 20 | ############################################################################################## 21 | 22 | eddyStampCheck <- function(tab, 23 | useFasttime=FALSE){ 24 | 25 | # convert time stamps 26 | tBgnErr <- FALSE 27 | tEndErr <- FALSE 28 | 29 | if(useFasttime) { 30 | tabBP <- try(fasttime::fastPOSIXct(tab$timeBgn, tz='GMT'), silent=T) 31 | tabEP <- try(fasttime::fastPOSIXct(tab$timeEnd, tz='GMT'), silent=T) 32 | } else { 33 | tabBP <- try(as.POSIXct(tab$timeBgn, format='%Y-%m-%dT%H:%M:%OS', tz='GMT'), silent=T) 34 | tabEP <- try(as.POSIXct(tab$timeEnd, format='%Y-%m-%dT%H:%M:%OS', tz='GMT'), silent=T) 35 | } 36 | 37 | if(any(c(inherits(tabBP,'try-error'), all(is.na(tabBP))))) { 38 | tBgnErr <- TRUE 39 | } 40 | 41 | if(any(c(inherits(tabBP,'try-error'), all(is.na(tabEP))))) { 42 | tEndErr <- TRUE 43 | } 44 | 45 | # if conversion failed, keep character time stamps and pass along message 46 | tabN <- tab 47 | err <- FALSE 48 | if(tBgnErr) { 49 | err <- TRUE 50 | } else { 51 | tabN$timeBgn <- tabBP 52 | } 53 | 54 | if(tEndErr) { 55 | err <- TRUE 56 | } else { 57 | tabN$timeEnd <- tabEP 58 | } 59 | 60 | # if conversion was successful, check for single-day empty records and remove 61 | if(err) { 62 | tabN <- tabN 63 | } else { 64 | dayDiff <- base::as.difftime(tabEP - tabBP) 65 | secDiff <- base::abs(base::as.numeric(dayDiff, units="secs")) 66 | dayDup <- which(secDiff >= 86399) 67 | if(length(dayDup)==0) { 68 | tabN <- tabN 69 | } else { 70 | tabN <- tabN[-dayDup,] 71 | } 72 | } 73 | 74 | return(list(tabN, err)) 75 | } 76 | 77 | -------------------------------------------------------------------------------- /neonUtilities/R/findDatatables.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Find data tables 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' List the names of the data tables within each folder 9 | #' 10 | #' @keywords internal 11 | #' @param folder The folder of the outputs 12 | #' @param fnames Full names - if true, then return the full file names including enclosing folders, if false, return only the file names 13 | #' @return a data frame of file names 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # Changelog and author contributions / copyrights 19 | # Christine Laney (2017-09-28) 20 | ############################################################################################## 21 | 22 | findDatatables <- function(folder, fnames = T){ 23 | fs <- list.dirs(folder) 24 | g <- grep(pattern = "stackedFiles", x = fs) 25 | if(length(g) > 0){fs <- fs[-g]} 26 | fls <- character() 27 | for(i in 1:length(fs)){ 28 | fls <- c(fls, list.files(fs[i], full.names = fnames)) 29 | } 30 | return(fls[which(substr(fls, nchar(fls)-3, nchar(fls)) == ".csv")]) 31 | } 32 | -------------------------------------------------------------------------------- /neonUtilities/R/findTablesByFormat.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Find unique data tables in dataset 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Find the unique data tables that are present in the dataset (e.g., 2 minute vs 30 minute, or pinning vs identification data) and their types, based on the file name formatting. Adapted from findTablesUnique(). 9 | #' 10 | #' @keywords internal 11 | #' @param datatables A list of data files 12 | #' @return An array of unique table names and their types 13 | 14 | #' @references 15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 16 | 17 | # Changelog and author contributions / copyrights 18 | # 2017-09-28 (Claire Lunch): created function, based on findTablesUnique() 19 | ############################################################################################## 20 | 21 | findTablesByFormat <- function(datatables){ 22 | dt <- basename(datatables) 23 | splitNames <- strsplit(x = dt, split = "\\.") 24 | t <- character() 25 | for (i in 1:length(splitNames)){ 26 | for(j in 3:length(splitNames[[i]])){ 27 | s <- splitNames[[i]][j] 28 | s <- gsub(x = s, pattern = "_pub", replacement = "") 29 | if(s != "sensor_positions" & s != "science_review_flags" & length(grep('_', s, fixed=T))>0) { 30 | t <- c(t, s) 31 | } 32 | } 33 | } 34 | 35 | if(length(t)==0) { 36 | stop("No data tables found, only metadata. Try downloading expanded package, and check availability on the NEON data portal.") 37 | } 38 | 39 | tn <- unique(t) 40 | tt <- character(length(tn)) 41 | 42 | for(k in 1:length(tn)) { 43 | names.k.all <- splitNames[union(grep(paste(".", tn[k], ".", sep=""), dt, fixed=T), 44 | grep(paste(".", tn[k], "_pub.", sep=""), dt, fixed=T))] 45 | tt.all <- lapply(names.k.all, function(xn) { 46 | if(length(which(xn==""))>0) { 47 | xn <- xn[-which(xn=="")] 48 | } 49 | if(length(xn)==5) { 50 | return("lab") 51 | } else { 52 | if(length(grep("[0-9]{4}-[0-9]{2}", xn))>0) { 53 | return("site-date") 54 | } 55 | else { 56 | return("site-all") 57 | } 58 | } 59 | }) 60 | tt.temp <- unique(unlist(tt.all)) 61 | if(length(tt.temp)>1) { 62 | stop(paste("In files to be stacked, table ", tn[k], 63 | " has been published under conflicting schedules. To avoid this problem, either work only with released data, or stack released and provisional data separately.", 64 | sep="")) 65 | } 66 | tt[k] <- tt.temp 67 | } 68 | 69 | tf <- cbind(tn,tt) 70 | tf <- data.frame(tf) 71 | names(tf) <- c("tableName","tableType") 72 | 73 | return(tf) 74 | } 75 | -------------------------------------------------------------------------------- /neonUtilities/R/findTablesUnique.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Find unique data tables in dataset 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Find the unique data tables that are present in the dataset (e.g., 2 minute vs 30 minute, or pinning vs identification data) 9 | #' 10 | #' @keywords internal 11 | #' @param datatables A list of data files 12 | #' @return An array of unique table names 13 | 14 | #' @references 15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 16 | 17 | # Changelog and author contributions / copyrights 18 | # 2017-09-28 (Christine Laney): created original function 19 | # 2018-04-13 (Christine Laney): updated to check for extra "_pub" appending table names, and remove 20 | ############################################################################################## 21 | 22 | findTablesUnique <- function(datatables, tabletypes){ 23 | dt <- datatables 24 | tt <- tabletypes 25 | splitNames <- strsplit(x = dt, split = "\\.") 26 | t <- character() 27 | for (i in 1:length(splitNames)){ 28 | for(j in 1:length(splitNames[[i]])){ 29 | s <- splitNames[[i]][j] 30 | s <- gsub(x = s, pattern = "_pub", replacement = "") 31 | if(s %in% tt$tableName){ 32 | t <- c(t, s) 33 | } 34 | } 35 | if(length(splitNames[[i]]) == 4){ 36 | n <- splitNames[[i]][3] 37 | n <- gsub(x = n, pattern = "_pub", replacement = "") 38 | t <- c(t, n) 39 | } 40 | if(length(splitNames[[i]]) %in% c(10,11)){ 41 | n <- splitNames[[i]][7] 42 | n <- gsub(x = n, pattern = "_pub", replacement = "") 43 | t <- c(t, n) 44 | } 45 | if(length(splitNames[[i]]) == 12){ 46 | n <- splitNames[[i]][11] 47 | n <- gsub(x = n, pattern = "_pub", replacement = "") 48 | t <- c(t, n) 49 | } 50 | if(length(splitNames[[i]]) == 14){ 51 | n <- splitNames[[i]][10] 52 | n <- gsub(x = n, pattern = "_pub", replacement = "") 53 | t <- c(t, n) 54 | } 55 | } 56 | return(unique(t)) 57 | } 58 | -------------------------------------------------------------------------------- /neonUtilities/R/getAPI.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get the data from API 3 | 4 | #' @author 5 | #' Nate Mietkiewicz \email{mietkiewicz@battelleecology.org} 6 | 7 | #' @description Accesses the API with options to use the user-specific API token generated within data.neonscience.org user accounts. 8 | #' 9 | #' @keywords internal 10 | #' @param apiURL The API endpoint URL 11 | #' @param token User specific API token (generated within data.neonscience.org user accounts). Optional. 12 | 13 | #' @references 14 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 15 | 16 | # Changelog and author contributions / copyrights 17 | # 2020-05-21 (Claire Lunch): Modified to check for reaching rate limit 18 | # 2020-03-21 (Nate Mietkiewicz): Created original function 19 | ############################################################################################## 20 | 21 | getAPI <- function(apiURL, token=NA_character_){ 22 | 23 | if(!curl::has_internet()) { 24 | message("No internet connection detected. Cannot access NEON API.") 25 | return(invisible()) 26 | } 27 | 28 | if(identical(token, "")) { 29 | token <- NA_character_ 30 | } 31 | 32 | usera <- paste("neonUtilities/", utils::packageVersion("neonUtilities"), " R/", 33 | R.Version()$major, ".", R.Version()$minor, " ", commandArgs()[1], 34 | " ", R.Version()$platform, sep="") 35 | 36 | if(is.na(token)) { 37 | 38 | # make 5 attempts to access - if rate limit is reached every time, give up 39 | j <- 1 40 | while(j < 6) { 41 | 42 | req <- try(httr::GET(apiURL, httr::user_agent(usera)), silent=T) 43 | 44 | # check for no response 45 | if(!inherits(req, "response")) { 46 | message("No response. NEON API may be unavailable, check NEON data portal for outage alerts. If the problem persists and can't be traced to an outage alert, check your computer for firewall or other security settings preventing R from accessing the internet.") 47 | return(invisible()) 48 | } 49 | 50 | # if rate limit is reached, pause 51 | if(!is.null(req$headers$`x-ratelimit-limit`)) { 52 | 53 | if(req$headers$`x-ratelimit-remaining`<=1) { 54 | message(paste("Rate limit reached. Pausing for ", 55 | req$headers$`x-ratelimit-reset`, 56 | " seconds to reset.", sep="")) 57 | Sys.sleep(req$headers$`x-ratelimit-reset`) 58 | j <- j+1 59 | } else { 60 | j <- j+5 61 | } 62 | } else { 63 | j <- j+5 64 | } 65 | } 66 | 67 | } else { 68 | 69 | # same process as in non-token case: make 5 attempts 70 | 71 | j <- 1 72 | while(j < 6) { 73 | 74 | req <- try(httr::GET(apiURL, httr::user_agent(usera), 75 | httr::add_headers(.headers = c('X-API-Token'= token, 76 | 'accept' = 'application/json'))), 77 | silent=T) 78 | 79 | # check for no response 80 | if(!inherits(req, "response")) { 81 | message("No response. NEON API may be unavailable, check NEON data portal for outage alerts. If the problem persists and can't be traced to an outage alert, check your computer for firewall or other security settings preventing R from accessing the internet.") 82 | return(invisible()) 83 | } 84 | 85 | # first check for null, since unlimited tokens don't have these headers 86 | if(!is.null(req$headers$`x-ratelimit-limit`)) { 87 | 88 | # if rate limit is reached, pause 89 | if(req$headers$`x-ratelimit-remaining`<=1) { 90 | message(paste("Rate limit reached. Pausing for ", 91 | req$headers$`x-ratelimit-reset`, 92 | " seconds to reset.", sep="")) 93 | Sys.sleep(req$headers$`x-ratelimit-reset`) 94 | j <- j+1 95 | } else { 96 | j <- j+5 97 | } 98 | } else { 99 | j <- j+5 100 | } 101 | } 102 | } 103 | 104 | return(req) 105 | 106 | } 107 | -------------------------------------------------------------------------------- /neonUtilities/R/getAPIHeaders.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get only headers from API 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description Accesses the API with options to use the user-specific API token generated within neon.datascience user accounts. 8 | #' 9 | #' @keywords internal 10 | #' @param apiURL The API endpoint URL 11 | #' @param token User specific API token (generated within data.neonscience.org user accounts). Optional. 12 | 13 | #' @references 14 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 15 | 16 | # Changelog and author contributions / copyrights 17 | # 2021-01-24 (Claire Lunch): Adapted from getAPI() 18 | ############################################################################################## 19 | 20 | getAPIHeaders <- function(apiURL, token=NA_character_){ 21 | 22 | if(!curl::has_internet()) { 23 | message("No internet connection detected. Cannot access NEON API.") 24 | return(invisible()) 25 | } 26 | 27 | if(identical(token, "")) { 28 | token <- NA_character_ 29 | } 30 | 31 | usera <- paste("neonUtilities/", utils::packageVersion("neonUtilities"), " R/", 32 | R.Version()$major, ".", R.Version()$minor, " ", commandArgs()[1], 33 | " ", R.Version()$platform, sep="") 34 | 35 | if(is.na(token)) { 36 | 37 | # make 5 attempts to access - if rate limit is reached every time, give up 38 | j <- 1 39 | while(j < 6) { 40 | 41 | req <- try(httr::HEAD(apiURL, httr::user_agent(usera)), silent=T) 42 | 43 | # check for no response 44 | if(!inherits(req, "response")) { 45 | message("No response. NEON API may be unavailable, check NEON data portal for outage alerts.") 46 | return(invisible()) 47 | } 48 | 49 | # if rate limit is reached, pause 50 | if(!is.null(req$headers$`x-ratelimit-limit`)) { 51 | 52 | if(req$headers$`x-ratelimit-remaining`<=1) { 53 | message(paste("Rate limit reached. Pausing for ", 54 | req$headers$`x-ratelimit-reset`, 55 | " seconds to reset.", sep="")) 56 | Sys.sleep(req$headers$`x-ratelimit-reset`) 57 | j <- j+1 58 | } else { 59 | j <- j+5 60 | } 61 | } else { 62 | j <- j+5 63 | } 64 | } 65 | 66 | } else { 67 | 68 | # same process as in non-token case: make 5 attempts 69 | 70 | j <- 1 71 | while(j < 6) { 72 | 73 | req <- try(httr::HEAD(apiURL, httr::user_agent(usera), 74 | httr::add_headers(.headers = c('X-API-Token'= token, 75 | 'accept' = 'application/json'))), 76 | silent=T) 77 | 78 | # check for no response 79 | if(!inherits(req, "response")) { 80 | message("No response. NEON API may be unavailable, check NEON data portal for outage alerts.") 81 | return(invisible()) 82 | } 83 | 84 | # first check for null, since unlimited tokens don't have these headers 85 | if(!is.null(req$headers$`x-ratelimit-limit`)) { 86 | 87 | # if rate limit is reached, pause 88 | if(req$headers$`x-ratelimit-remaining`<=1) { 89 | message(paste("Rate limit reached. Pausing for ", 90 | req$headers$`x-ratelimit-reset`, 91 | " seconds to reset.", sep="")) 92 | Sys.sleep(req$headers$`x-ratelimit-reset`) 93 | j <- j+1 94 | } else { 95 | j <- j+5 96 | } 97 | } else { 98 | j <- j+5 99 | } 100 | } 101 | } 102 | 103 | return(req) 104 | 105 | } 106 | -------------------------------------------------------------------------------- /neonUtilities/R/getAttributes.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Extract attributes from eddy covariance H5 files 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Extract attribute metadata from H5 files 9 | #' 10 | #' @param fil File path to the H5 file to extract attributes from [character] 11 | #' @param sit The site, for site attributes. Must match site of file path. [character] 12 | #' @param type The type of attributes to retrieve. [character] 13 | #' @param valName If CO2 validation metadata are requested, the H5 name of the level where they can be found. [character] 14 | 15 | #' @keywords internal 16 | #' @return A data frame with one row containing the extracted attributes 17 | 18 | #' @references 19 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | 21 | # changelog and author contributions / copyrights 22 | # Claire Lunch (2023-03-21) 23 | ############################################################################################## 24 | 25 | getAttributes <- function(fil, sit, type, valName) { 26 | 27 | if(grepl("basic", fil)) { 28 | mnth <- regmatches(fil, regexpr("20[0-9]{2}-[0-9]{2}", fil)) 29 | } else { 30 | mnth <- regmatches(fil, regexpr("20[0-9]{2}-[0-9]{2}-[0-9]{2}", fil)) 31 | } 32 | 33 | if(type=="site") { 34 | lev <- sit 35 | } 36 | if(type=="root") { 37 | lev <- "/" 38 | } 39 | if(type=="val") { 40 | lev <- valName 41 | } 42 | 43 | gAttr <- base::try(rhdf5::h5readAttributes(fil, name=lev), silent=T) 44 | 45 | if(inherits(gAttr, "try-error")) { 46 | gAttr <- data.frame(matrix(data=c(sit, mnth), ncol=2, nrow=1)) 47 | names(gAttr) <- c("site","date") 48 | } else { 49 | if(any(lapply(gAttr, length)>1)) { 50 | for(i in which(lapply(gAttr, length)>1)) { 51 | gAttr[i] <- paste0(gAttr[i], collapse=",") 52 | } 53 | } 54 | gAttr <- c(sit, mnth, gAttr) 55 | names(gAttr)[1:2] <- c("site", "date") 56 | } 57 | return(gAttr) 58 | } 59 | 60 | -------------------------------------------------------------------------------- /neonUtilities/R/getAvg.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get a list of the available averaging intervals for a data product 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Most IS products are available at multiple averaging intervals; get a list of what's available for a given data product 9 | #' 10 | #' @param dpID The identifier of the NEON data product, in the form DPL.PRNUM.REV, e.g. DP1.00006.001 11 | #' @param token User specific API token (generated within data.neonscience.org user accounts) 12 | 13 | #' @return A vector of the available averaging intervals, typically in minutes. 14 | 15 | #' @examples 16 | #' # Get available averaging intervals for PAR data product 17 | #' getAvg("DP1.00024.001") 18 | 19 | #' @references 20 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 21 | 22 | #' @export 23 | 24 | # changelog and author contributions / copyrights 25 | # Claire Lunch (2018-05-11) 26 | # original creation 27 | ############################################################################################## 28 | 29 | getAvg <- function(dpID, token = NA_character_) { 30 | 31 | message("getAvg() is deprecated; use getTimeIndex() in future", call.=FALSE) 32 | 33 | } 34 | -------------------------------------------------------------------------------- /neonUtilities/R/getCitation.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get a Bibtex citation for NEON data with a DOI, or generate a provisional Bibtex citation 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Use the DOI Foundation API to get Bibtex-formatted citations for NEON data, or use a template to generate a Bibtex citation for provisional data. Helper function to download and stacking functions. 9 | 10 | #' @param dpID The data product ID of the data to be cited [character] 11 | #' @param release The data release to be cited. Can be provisional. [character] 12 | 13 | #' @return A character string containing the Bibtex citation 14 | #' 15 | #' @export 16 | 17 | #' @examples 18 | #' \dontrun{ 19 | #' # Get the citation for Breeding landbird point counts (DP1.10003.001), RELEASE-2023 20 | #' cit <- getCitation(dpID="DP1.10003.001", release="RELEASE-2023") 21 | #' } 22 | 23 | #' @references 24 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 25 | 26 | # Changelog and author contributions / copyrights 27 | # Claire Lunch (2023-04-18) 28 | ############################################################################################## 29 | 30 | getCitation <- function(dpID=NA_character_, release=NA_character_) { 31 | 32 | if(is.na(dpID) | is.na(release)) { 33 | stop("dpID and release are required inputs.") 34 | } 35 | 36 | # if release=PROVISIONAL, construct provisional citation from template 37 | if(release=="PROVISIONAL") { 38 | 39 | cit <- cit_prov_template 40 | cit <- base::gsub(pattern="DPID", replacement=dpID, x=cit) 41 | cit <- base::gsub(pattern="YEAR", replacement=format(Sys.Date(), "%Y"), x=cit) 42 | 43 | # get product name from NEON API 44 | req <- httr::GET(paste("https://data.neonscience.org/api/v0/products/", 45 | dpID, sep="")) 46 | if(is.null(req)) {return(invisible())} 47 | if(!inherits(req, "response")) {return(invisible())} 48 | 49 | df <- jsonlite::fromJSON(httr::content(req, as="text", encoding="UTF-8"), 50 | simplifyDataFrame=TRUE, flatten=TRUE) 51 | nm <- df$data$productName 52 | cit <- base::gsub(pattern="NAME", replacement=nm, x=cit) 53 | 54 | } else { 55 | 56 | doi <- try(getNeonDOI(dpID=dpID, release=release), silent=TRUE) 57 | if(!inherits(doi, "try-error")) { 58 | 59 | req <- httr::GET(paste("https://doi.org/", doi$DOI, sep=""), 60 | httr::accept("application/x-bibtex")) 61 | 62 | # check for no or empty response 63 | if(is.null(req)) { 64 | message("No response. DOI Foundation API may be unavailable, check for outage alerts.") 65 | return(invisible()) 66 | } 67 | 68 | if(!inherits(req, "response")) { 69 | message("No response. DOI Foundation API may be unavailable, check for outage alerts.") 70 | return(invisible()) 71 | } 72 | 73 | # read response 74 | cit <- httr::content(req, as="text", encoding="UTF-8") 75 | 76 | } else { 77 | message(paste("DOI not found for data product ID ", dpID, "and release", 78 | release, ". Check inputs.", sep="")) 79 | return(invisible()) 80 | } 81 | 82 | } 83 | 84 | return(cit) 85 | 86 | } 87 | -------------------------------------------------------------------------------- /neonUtilities/R/getDatatable.R: -------------------------------------------------------------------------------- 1 | #' Get NEON data table 2 | #' 3 | #' 4 | #' @author Eric R. Sokol \email{esokol@battelleecology.org} 5 | #' 6 | #' 7 | #' @description This is a function to retrieve a data table 8 | #' from the NEON data portal for sites and dates provided by the 9 | #' enduser. NOTE that this function only works for NEON 10 | #' Observation System (OS) data products, and only for select tables 11 | #' 12 | #' @param dpid character sting for NEON data product ID 13 | #' @param data_table_name character sting for name of the data table to download, e.g., 'sls_soilCoreCollection' 14 | #' @param sample_location_list list of sites, domains, etc. If NA, retrieve all data for the given data table / dpid combination. 15 | #' @param sample_location_type character sting for location type, such as 'siteID'. Must be one of the NEON controlled terms. If you're unsure, use 'siteID' 16 | #' @param sample_date_min start date for query. Default is 1-Jan-2012, and this should capture the earliest NEON data record. 17 | #' @param sample_date_max end date for query. Default is current date. 18 | #' @param sample_date_format date format. Default/expected format is yyyy-mm-dd 19 | #' @param data_package_type package type, either 'basic' or 'expanded'. If unsure, use 'expanded' 20 | #' @param url_prefix_data data endpoint for NEON API. 21 | #' @param url_prefix_products products endpoint for NEON API. 22 | #' @param token User specific API token (generated within data.neonscience.org user accounts) 23 | #' 24 | #' @return data frame with selected NEON data 25 | #' 26 | #' 27 | #' @references License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 28 | #' 29 | #' 30 | #' @export 31 | getDatatable <- function( 32 | dpid = NA, #data product ID 33 | data_table_name = NA, #data table name 34 | sample_location_list = NA, # list of sites, domains, etc. 35 | sample_location_type = 'siteID', #location type 36 | sample_date_min = '2012-01-01', #start date -- default is 1-Jan-2012 37 | sample_date_max = Sys.Date(), #end date -- default is current date 38 | sample_date_format = '%Y-%m-%d', #date format 39 | ### more defaults 40 | data_package_type = 'basic', 41 | url_prefix_data = 'https://data.neonscience.org/api/v0/data/', 42 | url_prefix_products = 'https://data.neonscience.org/api/v0/products/', 43 | token = NA_character_){ 44 | 45 | message("getDatatable() is deprecated. Use loadByProduct() with input parameter tabl=") 46 | return(invisible()) 47 | 48 | } 49 | 50 | -------------------------------------------------------------------------------- /neonUtilities/R/getEddyLog.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get the full issue log set for the SAE bundle 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Use the NEON API to get the issue log from all products in the bundle in a user-friendly format 9 | 10 | #' @param token User specific API token (generated within data.neonscience.org user accounts) 11 | #' @keywords internal 12 | #' 13 | #' @return A table of issues reported for the data product. 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # Changelog and author contributions / copyrights 19 | # Claire Lunch (2021-10-27) 20 | ############################################################################################## 21 | getEddyLog <- function(token=NA_character_) { 22 | 23 | bundleDPs <- c("DP1.00007.001", "DP1.00010.001", "DP1.00034.001", "DP1.00035.001", 24 | "DP1.00036.001", "DP1.00037.001", "DP1.00099.001", "DP1.00100.001", 25 | "DP2.00008.001", "DP2.00009.001", "DP2.00024.001", "DP3.00008.001", 26 | "DP3.00009.001", "DP3.00010.001", "DP4.00002.001", "DP4.00007.001", 27 | "DP4.00067.001", "DP4.00137.001", "DP4.00201.001", "DP4.00200.001") 28 | 29 | eddyLog <- character() 30 | for(i in bundleDPs) { 31 | 32 | req <- getAPI(apiURL = paste0("https://data.neonscience.org/api/v0/products/", i), 33 | token = token) 34 | 35 | if(is.null(req)) { 36 | return(invisible()) 37 | } 38 | 39 | allproductinfo <- jsonlite::fromJSON(httr::content(req, as="text", encoding="UTF-8"), 40 | simplifyDataFrame=TRUE, flatten=TRUE) 41 | 42 | productlog <- allproductinfo[["data"]]$changeLogs 43 | 44 | if(!is.null(productlog)) { 45 | if(nrow(productlog)>0) { 46 | dataSubproduct <- rep(i, nrow(productlog)) 47 | productlog <- cbind(dataSubproduct, productlog) 48 | eddyLog <- rbind(eddyLog, productlog) 49 | } 50 | } 51 | 52 | } 53 | return(eddyLog) 54 | } 55 | -------------------------------------------------------------------------------- /neonUtilities/R/getFileUrls.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | #' Christine Laney \email{claney@battelleecology.org} 7 | 8 | #' @description Used to generate a data frame of available AOP files. 9 | #' 10 | #' @param m.urls The monthly API URL for the AOP files 11 | #' @param include.provisional T or F, should provisional data be included in downloaded files? 12 | #' @param token User specific API token (generated within data.neonscience.org user accounts) 13 | #' 14 | #' @keywords internal 15 | 16 | #' @return A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0) 17 | 18 | #' @references 19 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | 21 | # Changelog and author contributions / copyrights 22 | # Claire Lunch (2018-02-19): original creation 23 | # Christine Laney (2018-03-05): Added functionality to get new list of URLs if the old ones expire, during the download stream. 24 | 25 | ############################################################################################## 26 | 27 | # get and stash the file names, S3 URLs, file size, and download status (default = 0) in a data frame 28 | getFileUrls <- function(m.urls, include.provisional, token=NA){ 29 | url.messages <- character() 30 | file.urls <- c(NA, NA, NA) 31 | releases <- character() 32 | prov.message <- "no" 33 | for(i in 1:length(m.urls)) { 34 | 35 | tmp <- getAPI(apiURL = m.urls[i], token = token) 36 | 37 | if(tmp$status_code!=200) { 38 | message(paste("Data file retrieval failed with code ", tmp$status_code, 39 | ". Check NEON data portal for outage alerts.", sep="")) 40 | return(invisible()) 41 | } 42 | 43 | tmp.files <- jsonlite::fromJSON(httr::content(tmp, as='text', encoding='UTF-8'), 44 | simplifyDataFrame=TRUE, flatten=TRUE) 45 | 46 | # check for no files 47 | if(length(tmp.files$data$files)==0) { 48 | message(paste("No files found for site", tmp.files$data$siteCode, 49 | "and year", tmp.files$data$month, sep=" ")) 50 | next 51 | } 52 | 53 | # get release info 54 | if(isFALSE(include.provisional) & tmp.files$data$release=="PROVISIONAL") { 55 | file.urls <- file.urls 56 | releases <- releases 57 | prov.message <- "yes" 58 | } else { 59 | releases <- c(releases, tmp.files$data$release) 60 | 61 | file.urls <- rbind(file.urls, cbind(tmp.files$data$files$name, 62 | tmp.files$data$files$url, 63 | tmp.files$data$files$size)) 64 | } 65 | } 66 | 67 | if(all(is.na(file.urls)) & prov.message=="yes") { 68 | stop("All files found were provisional. Modify download query (dates and/or sites) or, if you want to use provisional data, use input parameter include.provisional=TRUE.") 69 | } 70 | 71 | if(prov.message=="yes") { 72 | message("Provisional data were excluded from available files list. To download provisional data, use input parameter include.provisional=TRUE.") 73 | } 74 | 75 | # get size info 76 | file.urls <- data.frame(file.urls, row.names=NULL) 77 | colnames(file.urls) <- c("name", "URL", "size") 78 | file.urls$URL <- as.character(file.urls$URL) 79 | file.urls$name <- as.character(file.urls$name) 80 | 81 | if(length(url.messages) > 0){writeLines(url.messages)} 82 | file.urls <- file.urls[-1,] 83 | release <- unique(releases) 84 | return(list(file.urls, release)) 85 | 86 | } 87 | -------------------------------------------------------------------------------- /neonUtilities/R/getIssueLog.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get the issue log for a specific data product 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Use the NEON API to get the issue log in a user-friendly format 9 | 10 | #' @param dpID The data product identifier, formatted as DP#.#####.### 11 | #' @param token User specific API token (generated within data.neonscience.org user accounts) 12 | 13 | #' @return A table of issues reported for the data product. 14 | 15 | #' @examples 16 | #' # Get documentation and availability of plant foliar properties data product 17 | #' cfcIssues <- getIssueLog("DP1.10026.001") 18 | 19 | #' @export 20 | 21 | #' @references 22 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 23 | 24 | # Changelog and author contributions / copyrights 25 | # Claire Lunch (2021-10-26) 26 | ############################################################################################## 27 | getIssueLog <- function(dpID=NA, token=NA_character_) { 28 | 29 | # error message if dpID isn't formatted as expected 30 | if(regexpr("DP[1-4]{1}.[0-9]{5}.00[0-9]{1}",dpID)[1]!=1) { 31 | stop(paste(dpID, "is not a properly formatted data product ID. The correct format is DP#.#####.00#", sep=" ")) 32 | } 33 | 34 | if(dpID=="DP4.00200.001") { 35 | issuelog <- getEddyLog(token=token) 36 | } else { 37 | req <- getAPI(apiURL = paste0("https://data.neonscience.org/api/v0/products/", dpID), 38 | token = token) 39 | 40 | if(is.null(req)) { 41 | return(invisible()) 42 | } 43 | 44 | allproductinfo <- jsonlite::fromJSON(httr::content(req, as="text", encoding="UTF-8"), 45 | simplifyDataFrame=TRUE, flatten=TRUE) 46 | 47 | issuelog <- allproductinfo[["data"]]$changeLogs 48 | } 49 | 50 | return(issuelog) 51 | } 52 | -------------------------------------------------------------------------------- /neonUtilities/R/getPackage.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get NEON data package 3 | #' 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Get a zipped file for a single data product, site, and year-month combination. Use the NEON data portal or API to determine data 9 | #' availability by data product, site, and year-month combinations. 10 | #' 11 | #' @param dpID The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001 12 | #' @param site_code A four-letter NEON research site code, such as HEAL for Healy. 13 | #' @param year_month The year and month of interest, in format YYYY-MM. 14 | #' @param package Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic. 15 | #' @param savepath The location to save the output files to 16 | #' @return A zipped monthly file 17 | 18 | #' @references 19 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | 21 | #' @export 22 | 23 | # Changelog and author contributions / copyrights 24 | # 2018-01-09 (Christine Laney): Created function 25 | # 2018-04-10 (Christine Laney): Exported function and added parameter for where to save the downloaded file. 26 | # 2022-02-07 (Claire Lunch): Original function relied on zip files, which are no longer stored. Redirected to zipsByProduct(), as first step to deprecating. 27 | ############################################################################################## 28 | 29 | getPackage <- function(dpID, site_code, year_month, package="basic", savepath = getwd()){ 30 | 31 | warning("getPackage() will be deprecated soon, use zipsByProduct(). For back-compatibility, your request has been sent to zipsByProduct().", 32 | call. = FALSE) 33 | zipsByProduct(dpID=dpID, site=site_code, startdate=year_month, 34 | enddate=year_month, package=package, 35 | savepath=savepath, check.size=FALSE) 36 | 37 | } 38 | -------------------------------------------------------------------------------- /neonUtilities/R/getProductInfo.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get NEON data product information 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Use the NEON API to get data product information such as availability, science team, etc. 9 | 10 | #' @importFrom jsonlite fromJSON 11 | #' @importFrom httr GET 12 | #' @param dpID The data product id (optional), formated as DP#.#####.### 13 | #' @param token User specific API token (generated within data.neonscience.org user accounts) 14 | 15 | #' @return A named list of metadata and availability information for a single data product. If the dpID argument is omitted, a table of information for all data products in the NEON catalog. 16 | 17 | #' @examples 18 | #' # Get documentation and availability of plant foliar properties data product 19 | #' cfcInfo <- getProductInfo("DP1.10026.001") 20 | 21 | #' @export 22 | 23 | #' @references 24 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 25 | 26 | # Changelog and author contributions / copyrights 27 | # Christine Laney (2017-10-01) 28 | ############################################################################################## 29 | getProductInfo <- function(dpID="", token = NA){ 30 | 31 | req <- getAPI(apiURL = paste0("https://data.neonscience.org/api/v0/products/", dpID), 32 | token = token) 33 | 34 | if(is.null(req)) { 35 | return(invisible()) 36 | } 37 | 38 | avail <- jsonlite::fromJSON(httr::content(req, as="text", encoding="UTF-8"), 39 | simplifyDataFrame=TRUE, flatten=TRUE) 40 | 41 | avail <- avail[["data"]] 42 | return(avail) 43 | } 44 | 45 | # query the products endpoint for the product requested 46 | -------------------------------------------------------------------------------- /neonUtilities/R/getProductSensors.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get data product-sensor relationships 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Pull all data from the NEON API /products endpoint, create a data frame with data product ID, data product name, and sensor type. 9 | 10 | #' @return A data frame 11 | 12 | #' @examples 13 | #' \dontrun{ 14 | #' sensors <- getProductSensors() 15 | #' } 16 | 17 | #' @export 18 | 19 | 20 | #' @references 21 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | 23 | # Changelog and author contributions / copyrights 24 | # 2018-04-24 Christine Laney: Created function 25 | ############################################################################################## 26 | 27 | getProductSensors <- function(){ 28 | products <- jsonlite::fromJSON("https://data.neonscience.org/api/v0/products", flatten=T) 29 | products <- products[["data"]] 30 | df <- data.frame(dpID = NA, dpName = NA, sensor = NA) 31 | for(i in 1:nrow(products)){ 32 | dpID <- products$productCode[i] 33 | dpName <- products$productName[i] 34 | sensor <- products$productSensor[i] 35 | df <- rbind(df, c(dpID, dpName, sensor)) 36 | } 37 | df <- df[-1, ] 38 | return(df) 39 | } 40 | 41 | -------------------------------------------------------------------------------- /neonUtilities/R/getReadmePublicationDate.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Scrape the publication date from each ReadMe file 3 | 4 | #' @author 5 | #' Nathan Mietkiewicz \email{mietkiewicz@battelleecology.org} 6 | 7 | #' @description 8 | #' Given a directory, this will recursively list all of the ReadMe files that were unzipped. 9 | #' This should result in a single text file with a list of all of the publication dates from the ReadMe file. 10 | 11 | #' @param savepath The root folder directory where the ReadMe files are located. 12 | #' @param out_filepath The output directory and filename. 13 | #' @param dpID The data product identifier 14 | #' 15 | #' @keywords internal 16 | 17 | #' @references 18 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 19 | 20 | # Changelog and author contributions / copyrights 21 | # 2019-10-08 (Nathan Mietkiewicz): Original creation 22 | 23 | ############################################################################################## 24 | getReadmePublicationDate <- function(savepath, out_filepath, dpID) { 25 | 26 | dpnum <- substring(dpID, 5, 9) 27 | out_filepath_name <- paste0(out_filepath, '/readme_', dpnum, '.txt') 28 | 29 | if(file.exists(out_filepath_name)) { 30 | unlink(out_filepath_name) 31 | } 32 | #writeLines("Stacking ReadMe documentation") 33 | readme_list <- list.files(savepath, pattern = '.readme.', 34 | recursive = TRUE, full.names = TRUE) 35 | if(length(readme_list)==0) { 36 | message("No readme file found") 37 | } else { 38 | 39 | readme_recent <- getRecentPublication(readme_list)[[1]] 40 | txt_file <- utils::read.delim(readme_recent, header=FALSE, quote="", stringsAsFactors=FALSE) 41 | txt_file <- txt_file$V1[grep("Date-Time", txt_file$V1, invert=TRUE)] 42 | 43 | tables <- table_types[which(table_types$productID==dpID),] 44 | if(nrow(tables)>0) { 45 | qInd <- grep('QUERY', txt_file) 46 | dPackInd <- grep('CONTENTS', txt_file) 47 | downPackInd <- grep('Basic download package', txt_file) 48 | 49 | txt_file[I(dPackInd+3)] <- paste('This data product contains up to', nrow(tables), 'data tables:') 50 | txt_file[I(dPackInd+5):I(dPackInd+4+nrow(tables))] <- paste(tables$tableName, tables$tableDesc, sep=' - ') 51 | txt_file[I(dPackInd+5+nrow(tables))] <- 'If data are unavailable for the particular sites and dates queried, some tables may be absent.' 52 | txt_file <- txt_file[-c(qInd:I(dPackInd-2), I(dPackInd+6+nrow(tables)):I(downPackInd-1))] 53 | } 54 | 55 | cat("###################################\n", file = out_filepath_name) 56 | cat("########### Disclaimer ############\n", file = out_filepath_name, append=TRUE) 57 | cat('This is the most recent readme publication based on all site-date combinations used during stackByTable.\nInformation specific to the query, including sites and dates, has been removed. The remaining content reflects general metadata for the data product.\n', file = out_filepath_name, append=TRUE) 58 | cat("##################################\n", file = out_filepath_name, append=TRUE) 59 | cat("\n", file = out_filepath_name, append=TRUE) 60 | utils::write.table(txt_file, out_filepath_name, append=TRUE, 61 | row.names=FALSE, col.names=FALSE, quote=FALSE) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /neonUtilities/R/getRecentPublication.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Returns the most recent files for those that do not need stacking 3 | 4 | #' @author 5 | #' Nathan Mietkiewicz \email{mietkiewicz@battelleecology.org} 6 | 7 | #' @description 8 | #' Given a list of files, this will order and return the file with the most recent publication date. 9 | 10 | #' @param inList The list of files. 11 | #' 12 | #' @keywords internal 13 | 14 | #' @references 15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 16 | 17 | # Changelog and author contributions / copyrights 18 | # 2019-10-08 (Nathan Mietkiewicz): Original creation 19 | # 2020-11-05 (Claire Lunch): Modify for compatibility with stackFromStore() 20 | 21 | ############################################################################################## 22 | getRecentPublication <- function(inList) { 23 | 24 | # if file dates are available, use file dates 25 | file_dates <- lapply(basename(inList), function(x) { 26 | regmatches(x, regexpr("[0-9]{8}T[0-9]{6}Z", x)) 27 | }) 28 | # if dates can't be found, move to dates in file path - for release 2021, these are all identical 29 | if(length(unlist(file_dates))==0) { 30 | 31 | path_dates <- lapply(inList, function(x) { 32 | regmatches(x, regexpr("[0-9]{8}T[0-9]{6}Z", x)) 33 | }) 34 | # if dates can't be found, use "max" of input file names. This should be very rare. 35 | if(length(unlist(path_dates))==0) { 36 | file_dates <- inList 37 | } else { 38 | file_dates <- path_dates 39 | } 40 | 41 | } 42 | 43 | outList <- inList[grep(max(unlist(file_dates)), inList)][1] 44 | return(list(outList, ifelse(nchar(file_dates[[1]])==16, max(unlist(file_dates)), 45 | "undetermined"))) 46 | } -------------------------------------------------------------------------------- /neonUtilities/R/getTaxonTable.R: -------------------------------------------------------------------------------- 1 | #' Get NEON taxon table 2 | #' 3 | #' 4 | #' @author Eric R. Sokol \email{esokol@battelleecology.org} 5 | #' 6 | #' 7 | #' @description This is a function to retrieve a taxon table 8 | #' from the NEON data portal for the taxon type by the 9 | #' enduser. 10 | #' 11 | #' @import httr jsonlite 12 | #' 13 | #' @param taxonType Character string for the taxonTypeCode. Must be one of ALGAE, BEETLE, BIRD, 14 | #' FISH, HERPETOLOGY, MACROINVERTEBRATE, MOSQUITO, MOSQUITO_PATHOGENS, SMALL_MAMMAL, PLANT, TICK 15 | #' @param recordReturnLimit Integer. The number of items to limit the result set to. If NA, will return all records in table. 16 | #' @param stream Character string, true or false. Option to obtain the result as a stream. Utilize for large requests. 17 | #' @param token User specific API token (generated within data.neonscience.org user accounts) 18 | #' 19 | #' @return data frame with selected NEON data 20 | #' 21 | #' 22 | #' @references License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 23 | #' 24 | #' 25 | #' @export 26 | 27 | getTaxonTable <- function( 28 | taxonType = NA, #string, taxonTypeCode, one of ALGAE, BEETLE, BIRD, FISH, HERPETOLOY, MACROINVERTEBRATE, MOSQUITO, MOSQUITO_PATHOGENS, SMALL_MAMMAL, PLANT, TICK 29 | recordReturnLimit = NA, #integer, The number of items to limit the result set to. If NA, will return all records in table. 30 | stream = 'true', #string, Option to obtain the result as a stream. Utilize for large requests. 31 | token = NA 32 | ){ 33 | 34 | message("getTaxonTable() is deprecated in neonUtilities. Use the updated version of this function, renamed to getTaxonList(), in the neonOS package.") 35 | 36 | } 37 | -------------------------------------------------------------------------------- /neonUtilities/R/getTileUrls.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | #' Christine Laney \email{claney@battelleecology.org} 7 | 8 | #' @description Produces a data frame that is populated by available tiles for the AOP product. 9 | #' 10 | #' @param m.urls The monthly API URL for the AOP tile. 11 | #' @param tileEasting A vector containing the easting UTM coordinates of the locations to download. 12 | #' @param tileNorthing A vector containing the northing UTM coordinates of the locations to download. 13 | #' @param include.provisional T or F, should provisional data be included in downloaded files? 14 | #' @param token User specific API token (generated within data.neonscience.org user accounts). Optional. 15 | #' 16 | #' @keywords internal 17 | #' 18 | #' @return A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0) 19 | 20 | #' @references 21 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | 23 | # Changelog and author contributions / copyrights 24 | # Claire Lunch (2018-02-19): original creation 25 | # Christine Laney (2018-03-05): Added functionality to get new list of URLs if the old ones expire, during the download stream. 26 | 27 | ############################################################################################## 28 | 29 | # get and stash the file names, S3 URLs, file size, and download status (default = 0) in a data frame 30 | getTileUrls <- function(m.urls, tileEasting, tileNorthing, 31 | include.provisional, token=NA_character_){ 32 | 33 | url.messages <- character() 34 | file.urls <- c(NA, NA, NA) 35 | releases <- character() 36 | prov.message <- "no" 37 | for(i in 1:length(m.urls)) { 38 | tmp <- getAPI(m.urls[i], token=token) 39 | if(tmp$status_code!=200) { 40 | message(paste("Data file retrieval failed with code ", tmp$status_code, 41 | ". Check NEON data portal for outage alerts.", sep="")) 42 | return(invisible()) 43 | } 44 | 45 | tmp.files <- jsonlite::fromJSON(httr::content(tmp, as="text", encoding="UTF-8"), 46 | simplifyDataFrame=T, flatten=T) 47 | 48 | # check for no files 49 | if(length(tmp.files$data$files)==0) { 50 | message(paste("No files found for site", tmp.files$data$siteCode, 51 | "and year", tmp.files$data$month, sep=" ")) 52 | next 53 | } 54 | 55 | # get release info 56 | if(isFALSE(include.provisional) & tmp.files$data$release=="PROVISIONAL") { 57 | file.urls <- file.urls 58 | releases <- releases 59 | prov.message <- "yes" 60 | } else { 61 | releases <- c(releases, tmp.files$data$release) 62 | 63 | # filter to only files for the relevant tiles 64 | ind <- numeric() 65 | for(j in 1:length(tileEasting)) { 66 | ind.j <- intersect(grep(paste('_', tileEasting[j], '_', sep=''), tmp.files$data$files$name), 67 | grep(paste('_', tileNorthing[j], '_', sep=''), tmp.files$data$files$name)) 68 | if(length(ind.j)>0) { 69 | ind <- c(ind, ind.j) 70 | } else { 71 | message(paste("No tiles found for easting ", 72 | tileEasting[j], " and northing ", 73 | tileNorthing[j], sep="")) 74 | } 75 | } 76 | ind <- unique(ind) 77 | tile.files <- tmp.files$data$files[ind,] 78 | 79 | file.urls <- rbind(file.urls, cbind(tile.files$name, 80 | tile.files$url, 81 | tile.files$size)) 82 | } 83 | 84 | } 85 | 86 | if(all(is.na(file.urls)) & prov.message=="yes") { 87 | stop("All files found were provisional. Modify download query (dates and/or sites) or, if you want to use provisional data, use input parameter include.provisional=TRUE.") 88 | } 89 | 90 | if(prov.message=="yes") { 91 | message("Provisional data were excluded from available files list. To download provisional data, use input parameter include.provisional=TRUE.") 92 | } 93 | 94 | # get size info 95 | file.urls <- data.frame(file.urls, row.names=NULL) 96 | colnames(file.urls) <- c("name", "URL", "size") 97 | file.urls$URL <- as.character(file.urls$URL) 98 | file.urls$name <- as.character(file.urls$name) 99 | 100 | if(length(url.messages) > 0){writeLines(url.messages)} 101 | file.urls <- file.urls[-1,] 102 | release <- unique(releases) 103 | return(list(file.urls, release)) 104 | 105 | } 106 | -------------------------------------------------------------------------------- /neonUtilities/R/getTimeIndex.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get a list of the available time intervals for a data product 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Most IS products are available at multiple time intervals; get a list of what's available for a given data product 9 | #' 10 | #' @param dpID The identifier of the NEON data product, in the form DPL.PRNUM.REV, e.g. DP1.00006.001 11 | #' @param token User specific API token (generated within data.neonscience.org user accounts) 12 | 13 | #' @return A vector of the available time intervals, typically in minutes. 14 | 15 | #' @examples 16 | #' # Get available time intervals for PAR data product 17 | #' getTimeIndex("DP1.00024.001") 18 | 19 | #' @references 20 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 21 | 22 | #' @export 23 | 24 | # changelog and author contributions / copyrights 25 | # Claire Lunch (2020-09-02) 26 | # duplication of getAvg() to match new nomenclature 27 | ############################################################################################## 28 | 29 | getTimeIndex <- function(dpID, token = NA_character_) { 30 | 31 | # error message if dpID isn't formatted as expected 32 | if(regexpr("DP[1-4]{1}.[0-9]{5}.[0-9]{3}",dpID)!=1) { 33 | stop(paste(dpID, "is not a properly formatted data product ID. The correct format is DP#.#####.###", sep=" ")) 34 | } 35 | 36 | # error message if dpID is the EC product 37 | if(dpID=="DP4.00200.001") { 38 | stop(paste(dpID, "is the bundled eddy covariance data product; download by averaging interval is not available.", sep=" ")) 39 | } 40 | 41 | # error message if dpID isn't in table_types 42 | if(!dpID %in% table_types$productID) { 43 | stop(paste(dpID, "is not a supported data product ID. Check for typos.", sep=" ")) 44 | } 45 | 46 | # error message if dpID isn't IS 47 | req <- getAPI(paste(apiURL = "https://data.neonscience.org/api/v0/products/", dpID, sep=""), 48 | token = token) 49 | 50 | if(is.null(req)) { 51 | message(paste("Could not verify data product. NEON API may be unavailable.")) 52 | return(invisible()) 53 | } 54 | 55 | avail <- jsonlite::fromJSON(httr::content(req, as='text', encoding='UTF-8'), 56 | simplifyDataFrame=TRUE, flatten=TRUE) 57 | 58 | if(avail$data$productScienceTeamAbbr %in% c("TOS","AOS","AOP") | 59 | dpID %in% c("DP1.20267.001","DP1.00101.001","DP1.00013.001","DP1.00038.001", 60 | "DP1.00096.001","DP1.00097.001","DP4.00133.001")) { 61 | stop(paste(dpID, "is not a streaming sensor (IS) data product; averaging interval is not relevant.", sep=" ")) 62 | } 63 | 64 | # look up TMI in table_types 65 | tmi <- table_types$tableTMI[which(table_types$productID==dpID)] 66 | return(tmi) 67 | 68 | } 69 | -------------------------------------------------------------------------------- /neonUtilities/R/getTitle.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get NEON data product title 3 | #' 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Create a title for a NEON data CSV file 9 | #' 10 | #' @keywords internal 11 | 12 | #' @param filename A NEON file name 13 | #' @return A title for the respective GeoCSV file 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # Changelog and author contributions / copyrights 19 | # 2018-01-09 (Christine Laney): created function 20 | ############################################################################################## 21 | 22 | getTitle <- function(filename) { 23 | splitName <- strsplit(x = filename, split = "\\.") 24 | if(length(splitName[[1]])==2){ 25 | tableName <- splitName[[1]][1] 26 | title <- paste0("NEON ", tableName, " data from multiple sites and/or months") 27 | } 28 | if(length(splitName[[1]]) > 2){ 29 | site <- splitName[[1]][3] 30 | dpID <- substr(filename, 15, 27) 31 | hor <- splitName[[1]][7] 32 | ver <- splitName[[1]][8] 33 | time <- paste(splitName[[1]][9], " minute resolution") #this will need updating for non met data products 34 | ym <- splitName[[1]][11] 35 | ym <- as.Date(x = paste0(ym, "-01"), format = "%Y-%m-%d") 36 | y <- format(ym, "%Y") 37 | m <- format(ym, "%B") 38 | uri <- paste0("https://data.neonscience.org/api/v0/products/", dpID) 39 | data_info <- jsonlite::fromJSON(txt = uri) 40 | dpName <- data_info$data$productName 41 | title <- paste("NEON", dpName, "data from", site, m, y, "at horizontal level", hor, "and vertical position", ver, sep = " " ) 42 | } 43 | return(title) 44 | } 45 | -------------------------------------------------------------------------------- /neonUtilities/R/getVariables.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get correct data types 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Support way to force R to read assign correct data types to each column based on variables file 9 | #' 10 | #' @keywords internal 11 | #' @param varFile A file that contains variable definitions 12 | #' @return A data frame with fieldName and assigned column class, along with table if present 13 | 14 | #' @references 15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 16 | 17 | # Changelog and author contributions / copyrights 18 | # Christine Laney (2017-09-28) 19 | ############################################################################################## 20 | 21 | getVariables <- function(varFile){ 22 | d <- utils::read.csv(varFile, header = T, stringsAsFactors = F) # read in a variables file 23 | d$colClass <- rep("numeric", nrow(d)) # make a new colClass column defaulting to numeric 24 | d$colClass[which(d$dataType %in% c("string", "date", "dateTime", "uri"))] <- "character" # change to character if string or dateTime 25 | if("table" %in% names(d)){ # OS variables files have table, IS do not 26 | return(d[, c("table", "fieldName", "colClass")]) 27 | } 28 | return(d[, c("fieldName","colClass")]) 29 | } 30 | -------------------------------------------------------------------------------- /neonUtilities/R/getVariablesEddy.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get variable names and units from SAE H5 files 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Extract variable names and units from SAE H5 files and return in user-friendly form. Used in stackEddy(), not intended for independent use. 9 | #' 10 | #' @keywords internal 11 | #' @param tabList A list of SAE data tables 12 | #' @return A table of variable names and units, aggregated from the input tables 13 | 14 | #' @references 15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 16 | 17 | # Changelog and author contributions / copyrights 18 | # Claire Lunch (2023-05-30) 19 | ############################################################################################## 20 | 21 | getVariablesEddy <- function(tabList){ 22 | 23 | variables <- character(5) 24 | for(p in 1:length(tabList[[1]])) { 25 | if(!is.null(attributes(tabList[[1]][[p]])$unit)) { 26 | var.nm <- strsplit(names(tabList[[1]])[p], 27 | split="/", fixed=T)[[1]][c(3,4,length(strsplit(names(tabList[[1]])[p], 28 | split="/", fixed=T)[[1]]))] 29 | if(length(attributes(tabList[[1]][[p]])$unit)>1) { 30 | var.nm <- matrix(var.nm, ncol=3, nrow=length(attributes(tabList[[1]][[p]])$unit), byrow=T) 31 | if(length(attributes(tabList[[1]][[p]])$unit)==length(attributes(tabList[[1]][[p]])$names)) { 32 | var.nm <- cbind(var.nm, attributes(tabList[[1]][[p]])$names) 33 | } else { 34 | if("index" %in% attributes(tabList[[1]][[p]])$names) { 35 | var.nm <- cbind(var.nm, 36 | attributes(tabList[[1]][[p]])$names[-which(attributes(tabList[[1]][[p]])$names=="index")]) 37 | } else { 38 | var.nm <- cbind(var.nm, 39 | attributes(tabList[[1]][[p]])$names[-which(attributes(tabList[[1]][[p]])$names 40 | %in% c("timeBgn","timeEnd"))]) 41 | } 42 | } 43 | var.nm <- cbind(var.nm, attributes(tabList[[1]][[p]])$unit) 44 | variables <- rbind(variables, var.nm) 45 | } else { 46 | variables <- rbind(variables, c(var.nm, "", attributes(tabList[[1]][[p]])$unit)) 47 | } 48 | } 49 | } 50 | variables <- data.frame(unique(variables)) 51 | 52 | if(nrow(variables)==1) { 53 | variables <- NA 54 | } else { 55 | variables <- variables[-1,] 56 | colnames(variables) <- c("category","system","variable","stat","units") 57 | rownames(variables) <- 1:nrow(variables) 58 | } 59 | 60 | return(variables) 61 | } 62 | 63 | -------------------------------------------------------------------------------- /neonUtilities/R/getVarsEddy.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Extract list of eddy covariance tables from HDF5 files 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Extracts a list of table metadata from a single HDF5 file. Specific to eddy covariance data product: DP4.00200.001. Can inform inputs to stackEddy(); variables listed in 'name' are available inputs to the 'var' parameter in stackEddy(). 9 | #' 10 | #' @param filepath The folder containing the H5 file [character] 11 | 12 | #' @return A data frame of the metadata for each data table in the HDF5 file 13 | 14 | #' @examples 15 | #' \dontrun{ 16 | #' # read variables from a file in a hypothetical filepath 17 | #' ec.vars <- getVarsEddy(filepath='/data/NEON.D19.BONA.DP4.00200.001.nsae.2017-12.basic.h5') 18 | #' } 19 | 20 | #' @references 21 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | 23 | #' @export 24 | 25 | # changelog and author contributions / copyrights 26 | # Claire Lunch (2019-06-07) 27 | # partially adapted from eddy4R.base::def.hdf5.extr() authored by David Durden 28 | ############################################################################################## 29 | 30 | getVarsEddy <- function(filepath) { 31 | 32 | # first check for rhdf5 package 33 | if(!requireNamespace("rhdf5", quietly=T)) { 34 | stop("Package rhdf5 is required for this function to work. 35 | \nrhdf5 is a Bioconductor package. To install, use:\ninstall.packages('BiocManager')\nBiocManager::install('rhdf5')\n") 36 | } 37 | 38 | listObj <- base::try(rhdf5::h5ls(filepath), silent=T) 39 | 40 | if(inherits(listObj, "try-error")) { 41 | stop(paste(filepath, " could not be read.", sep="")) 42 | } 43 | 44 | listDataObj <- listObj[listObj$otype == "H5I_DATASET",] 45 | 46 | listObjSpl <- tidyr::separate(listDataObj, col="group", 47 | into=c(NA, "site", "level", "category", "system", 48 | "horvertmi", "subsys"), 49 | sep="/", fill="right") 50 | listObjSpl <- tidyr::separate(listObjSpl, col="horvertmi", 51 | into=c("hor", "ver", "tmi"), 52 | sep="_", fill="left") 53 | listObjSpl$oth[which(is.na(suppressWarnings(as.numeric(listObjSpl$ver))))] <- 54 | listObjSpl$ver[which(is.na(suppressWarnings(as.numeric(listObjSpl$ver))))] 55 | listObjSpl$ver[which(is.na(suppressWarnings(as.numeric(listObjSpl$ver))))] <- NA 56 | 57 | return(listObjSpl) 58 | 59 | } 60 | -------------------------------------------------------------------------------- /neonUtilities/R/listFilesInZip.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get a data frame with the names of all files within a zipped NEON data package 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Given the top level zip file, return dataframe of all of the files within it without 9 | #' unzipping the file 10 | 11 | #' @param zippath The path to a zip file 12 | #' @return A list of filenames within the given zip file 13 | #' @keywords internal 14 | #' 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # changelog and author contributions / copyrights 19 | # Christine Laney (2017-07-02) 20 | ############################################################################################## 21 | 22 | listFilesInZip <- function(zippath){ 23 | utils::unzip(zipfile = zippath, list = T) 24 | } 25 | -------------------------------------------------------------------------------- /neonUtilities/R/listZipfiles.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Get all zip file names within a zipped NEON data package 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Given the data frame of all the files within the top level zip file, 9 | #' return an array of just the zip file names (no pdf, xml, or other files). 10 | #' 11 | #' @param zippath The path to a zip file 12 | #' @return An array of all zip files contained within the focal zip file 13 | #' @keywords internal 14 | #' 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # changelog and author contributions / copyrights 19 | # Christine Laney (2017-07-02) 20 | ############################################################################################## 21 | 22 | listZipfiles <- function(zippath){ 23 | df <- utils::unzip(zipfile = zippath, list = T) 24 | ns <- df[,1] 25 | fn <- ns[which(substr(ns, nchar(ns)-3, nchar(ns)) == ".zip")] 26 | return(fn) 27 | } 28 | -------------------------------------------------------------------------------- /neonUtilities/R/makePosColumns.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Create position (horizontal and vertical) columns 3 | 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' For instrumented meteorological data products, create position (horizontal and vertical) columns based on values 9 | #' embedded in the file names. 10 | #' 11 | #' @keywords internal 12 | #' @param d A data table 13 | #' @param datafl A data file name 14 | #' @return A data table with new columns 15 | 16 | #' @references 17 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 18 | 19 | # Changelog and author contributions / copyrights 20 | # 2017-09-28 (Christine Laney): Created original function 21 | # 2018-04-13 (Christine Laney): 22 | # * Continuous stream discharge (DP4.00130.001) is an OS product in IS format, and the HOR.VER 23 | # values given (100.100) are always the same. New HOR and VER columns are not needed in the 24 | # stacked product. 25 | # 2019-10-17 (Nathan Mietkiewicz): Add domainID, siteID, and collection YYYY-MM columns for sensor position files 26 | ############################################################################################## 27 | 28 | makePosColumns <- function(d, datafl, site){ 29 | 30 | datafl.splitFile <- strsplit(x = datafl, split = "\\/") 31 | datafl.splitName <- strsplit(x = datafl.splitFile[[1]][length(datafl.splitFile[[1]])], split = "\\.") 32 | 33 | sensor_positions <- grepl('sensor_positions', datafl.splitName) 34 | 35 | if((datafl.splitName[[1]][4]=="DP4") && (datafl.splitName[[1]][5]=="00130") & 36 | FALSE %in% sensor_positions){return(d)} 37 | 38 | nc <- ncol(d) 39 | if(length(datafl.splitName[[1]]) %in% c(12,14) || (TRUE %in% sensor_positions)){ 40 | if(length(datafl.splitName[[1]]) == 12){ 41 | horPos <- 8 42 | verPos <- 9 43 | } 44 | if(length(datafl.splitName[[1]]) == 14){ 45 | horPos <- 7 46 | verPos <- 8 47 | } 48 | if(TRUE %in% sensor_positions) { 49 | d$siteID <- rep(site, nrow(d)) 50 | d <- data.table::setcolorder(d, c(ncol(d), 1:I(ncol(d)-1))) 51 | } else { 52 | if(!("siteID" %in% names(d))){ 53 | d$domainID <- as.character(unlist(datafl.splitName)[2]) 54 | d$siteID <- as.character(unlist(datafl.splitName)[3]) 55 | } 56 | d$horizontalPosition <- as.character(rep(as.character(datafl.splitName[[1]][horPos]), nrow(d))) 57 | d$verticalPosition <- as.character(rep(as.character(datafl.splitName[[1]][verPos]), nrow(d))) 58 | d <- data.table::setcolorder(d, c((nc+1):(nc+4),1:nc)) 59 | } 60 | } 61 | d$publicationDate <- as.character(rep(datafl.splitName[[1]][length(datafl.splitName[[1]])-1]), nrow(d)) 62 | return(d) 63 | } -------------------------------------------------------------------------------- /neonUtilities/R/other_bundles.R: -------------------------------------------------------------------------------- 1 | #' Bundled vegetation and sediment data product information 2 | #' 3 | #' A dataset containing NEON data product codes of vegetation and sediment data products and the "home" data products they are bundled with. 4 | #' 5 | #' @format A data frame with 2 variables: 6 | #' \describe{ 7 | #' \item{product}{Data product ID of a product} 8 | #' \item{homeProduct}{Data product ID of the corresponding home data product} 9 | #' } 10 | #' @source NEON data product bundles 11 | "other_bundles" 12 | -------------------------------------------------------------------------------- /neonUtilities/R/quietMessages.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Will suppress all output messages, while retaining the output dataframe 3 | 4 | #' @author 5 | #' Nate Mietkiewicz \email{mietkiewicz@battelleecology.org} 6 | 7 | #' @description Used to quiet all output messages 8 | #' 9 | #' @param toBeQuieted Input to be quieted 10 | 11 | #' @return The expected output without associated messages/warnings. 12 | #' @keywords internal 13 | 14 | #' @references 15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 16 | 17 | # Changelog and author contributions / copyrights 18 | # Nate Mietkiewcz (2020-02-27): original creation 19 | 20 | ############################################################################################## 21 | 22 | quietMessages <- function(toBeQuieted) { 23 | sink(tempfile()) 24 | on.exit(sink()) 25 | invisible(force(toBeQuieted)) 26 | } 27 | -------------------------------------------------------------------------------- /neonUtilities/R/readTableNEON.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Read a NEON data table with correct data types for each variable 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Load a table into R, assigning classes to each column based on data types in variables file; or convert a table already loaded 9 | #' 10 | #' @param dataFile A data frame containing a NEON data table, or the filepath to a data table to load 11 | #' @param varFile A data frame containing the corresponding NEON variables file, or the filepath to the variables file 12 | #' @param useFasttime Should the fasttime package be used to read date-time variables? Defaults to false. 13 | #' @return A data frame of a NEON data table, with column classes assigned by data type 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | #' 18 | #' @export 19 | 20 | # Changelog and author contributions / copyrights 21 | # Claire Lunch (2019-11-08) 22 | ############################################################################################## 23 | 24 | readTableNEON <- function(dataFile, varFile, useFasttime=FALSE){ 25 | 26 | # check for fasttime package, if used 27 | if(useFasttime & !requireNamespace("fasttime", quietly=T)) { 28 | stop("Parameter useFasttime is TRUE but fasttime package is not installed. Install and re-try.") 29 | } 30 | 31 | # read in variables file 32 | if(inherits(varFile, 'character')) { 33 | v <- utils::read.csv(varFile, header = T, stringsAsFactors = F, 34 | na.strings=c('NA',''), encoding='UTF-8') 35 | } else { 36 | v <- try(as.data.frame(varFile), silent=T) 37 | if(inherits(v, 'try-error')) { 38 | stop('varFile must be either a NEON variables table or a file path to a NEON variables table.\n') 39 | } 40 | } 41 | 42 | # check this is a valid variables file 43 | if(any(c('category','system','stat') %in% names(v))) { 44 | stop('varFile appears to match DP4.00200.001. Use the stackEddy() function to work with surface-atmosphere exchange data.') 45 | } else { 46 | if(any(!c('table','fieldName','dataType') %in% names(v))) { 47 | stop('varFile is not a variables file, or is missing critical values.\n') 48 | } 49 | } 50 | 51 | # make a new colClass column defaulting to numeric 52 | # modify to character for strings and urls 53 | v$colClass <- rep("numeric", nrow(v)) 54 | v$colClass[which(v$dataType %in% c("string","uri"))] <- "character" 55 | v$colClass[which(v$dataType %in% c("date","dateTime"))] <- 'date' 56 | 57 | v <- v[, c("table", "fieldName", "colClass")] 58 | 59 | # read in data file 60 | if(inherits(dataFile, 'character')) { 61 | d <- suppressWarnings(utils::read.csv(dataFile, header=T, stringsAsFactors=F, 62 | colClasses=c(horizontalPosition='character', verticalPosition='character'), 63 | na.strings=c('NA',''), encoding='UTF-8')) 64 | } else { 65 | d <- try(as.data.frame(dataFile, stringsAsFactors=F), silent=T) 66 | if(inherits(d, 'try-error')) { 67 | stop('dataFile must be either a NEON data table or a file path to a NEON data table.\n') 68 | } 69 | } 70 | 71 | # check that most fields have a corresponding value in variables 72 | m <- length(which(!names(d) %in% v$fieldName)) 73 | if(m==length(names(d))) { 74 | stop('Variables file does not match data file.\n') 75 | } 76 | if(m>4) { 77 | message(paste(m, " fieldNames are present in data files but not in variables file. Unknown fields are read as character strings.\n", sep="")) 78 | } 79 | 80 | # fieldNames each have a unique dataType - don't need to match table 81 | for(i in names(d)) { 82 | if(!i %in% v$fieldName) { 83 | d[,i] <- as.character(d[,i]) 84 | } else { 85 | type <- v$colClass[which(v$fieldName==i)][1] 86 | if(type=='numeric') { 87 | d[,i] <- as.numeric(d[,i]) 88 | } 89 | if(type=='character') { 90 | d[,i] <- as.character(d[,i]) 91 | } 92 | if(type=='date') { 93 | d[,i] <- dateConvert(d[,i], useFasttime=useFasttime) 94 | } 95 | } 96 | } 97 | 98 | return(d) 99 | } 100 | -------------------------------------------------------------------------------- /neonUtilities/R/schemaAllStrings.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Create an arrow schema with every variable coded as a string field. 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Use the field names in a NEON variables file to create an arrow schema of all strings, or, if no variables file is available, read the header and assign everything as string. 9 | #' 10 | #' @param variables A data frame containing a NEON variables file for a single table, or a set of field names. 11 | #' @keywords internal 12 | #' 13 | #' @return An arrow schema for the relevant data table with all variables set to string. 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | # Changelog and author contributions / copyrights 19 | # Claire Lunch (2025-03-12) 20 | ############################################################################################## 21 | 22 | schemaAllStrings <- function(variables) { 23 | 24 | # still need to build in option for set of names, without full variables file 25 | # schemaFromVar() is now set to only call this function if the variables file is available 26 | # so only need the names option if this function is called from elsewhere in the future 27 | stringschema <- arrow::schema(unlist(apply(variables, MARGIN=1, FUN=function(x) { 28 | arrow::field(name=x["fieldName"], type=arrow::string()) 29 | }))) 30 | 31 | return(stringschema) 32 | 33 | } 34 | 35 | -------------------------------------------------------------------------------- /neonUtilities/R/schemaFromVar.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Create an arrow schema from a NEON variables file. 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Use the field names and data types in a NEON variables file to create an arrow schema. 9 | #' 10 | #' @param variables A data frame containing a NEON variables file, or a url pointing to a NEON variables file. 11 | #' @param tab The name of the table to generate a schema from. 12 | #' @param package Should the schema be created for the basic or expanded package? 13 | #' 14 | #' @return An arrow schema for the relevant data table. 15 | #' 16 | #' @export 17 | 18 | #' @references 19 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | 21 | # Changelog and author contributions / copyrights 22 | # Claire Lunch (2025-03-11) 23 | ############################################################################################## 24 | 25 | schemaFromVar <- function(variables, tab, package) { 26 | 27 | unify <- FALSE 28 | 29 | # is the input a file or a url? 30 | if(inherits(variables, "character")) { 31 | # read in variables file 32 | vartab <- try(data.frame(arrow::read_csv_arrow(variables)), silent=TRUE) 33 | vartab <- try(vartab[which(vartab$table==tab),], silent=TRUE) 34 | if(inherits(vartab, "try-error")) { 35 | message("There was a problem reading the variables file. Data types will be inferred.") 36 | unify <- TRUE 37 | } else { 38 | if(nrow(vartab)==0) { 39 | message("There was a problem reading the variables file. Data types will be inferred.") 40 | unify <- TRUE 41 | } 42 | } 43 | } else { 44 | vartab <- try(variables[which(variables$table==tab),], silent=TRUE) 45 | if(inherits(vartab, "try-error")) { 46 | message("There was a problem reading the variables file. Data types will be inferred.") 47 | unify <- TRUE 48 | } 49 | } 50 | 51 | # if reading the file failed, function returns NULL 52 | if(unify) { 53 | vschema <- NULL 54 | } else { 55 | 56 | # if working with the basic package, subset the table 57 | if(package=="basic") { 58 | vartab <- vartab[which(vartab$downloadPkg=="basic"),] 59 | } 60 | 61 | # start by making a schema with everything as a string 62 | vschema <- schemaAllStrings(vartab) 63 | 64 | # translate data types to arrow types for non-character fields 65 | ind <- which(!vartab$dataType %in% c("string", "uri")) 66 | for(i in ind) { 67 | if(vartab$dataType[i]=="real") { 68 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i], type=arrow::float64()) 69 | } 70 | if(vartab$dataType[i] %in% c("integer", "unsigned integer", 71 | "signed integer")) { 72 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i], type=arrow::int64()) 73 | } 74 | if(vartab$dataType[i]=="dateTime" & 75 | vartab$pubFormat[i] %in% c("yyyy-MM-dd'T'HH:mm:ss'Z'(floor)", 76 | "yyyy-MM-dd'T'HH:mm:ss'Z'", 77 | "yyyy-MM-dd'T'HH:mm:ss'Z'(round)")) { 78 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i], 79 | type=arrow::timestamp("s", timezone="UTC")) 80 | } 81 | if(vartab$dataType[i]=="dateTime" & 82 | vartab$pubFormat[i] %in% c("yyyy-MM-dd(floor)", "yyyy-MM-dd")) { 83 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i], type=arrow::date32()) 84 | } 85 | if(vartab$dataType[i]=="dateTime" & 86 | vartab$pubFormat[i] %in% c("yyyy(floor)", "yyyy(round)")) { 87 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i], type=arrow::int64()) 88 | } 89 | 90 | } 91 | 92 | } 93 | 94 | return(vschema) 95 | 96 | } 97 | 98 | -------------------------------------------------------------------------------- /neonUtilities/R/shared_aquatic.R: -------------------------------------------------------------------------------- 1 | #' Terrestrial-aquatic shared data information 2 | #' 3 | #' A dataset containing NEON site codes and data product IDs for places where meteorological data from terrestrial sites are used as the data of record for nearby aquatic sites as well. 4 | #' 5 | #' @format A data frame with 3 variables: 6 | #' \describe{ 7 | #' \item{site}{site code of a NEON aquatic site} 8 | #' \item{towerSite}{site code of the NEON terrestrial site used as the data source for the corresponding aquatic site} 9 | #' \item{product}{Data product ID of the data products to which the corresponding terrestrial-aquatic site relationship relates} 10 | #' } 11 | #' @source NEON site layouts and spatial design 12 | "shared_aquatic" 13 | -------------------------------------------------------------------------------- /neonUtilities/R/shared_flights.R: -------------------------------------------------------------------------------- 1 | #' Flight coverage information 2 | #' 3 | #' A dataset containing NEON site codes for places where a single AOP flight may cover multiple sites 4 | #' 5 | #' @format A data frame with 2 variables: 6 | #' \describe{ 7 | #' \item{site}{site code of a NEON site} 8 | #' \item{flightSite}{site code that matches the file naming for flights that may include "site"} 9 | #' } 10 | #' @source NEON flight plans 11 | "shared_flights" 12 | -------------------------------------------------------------------------------- /neonUtilities/R/sysdata.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/R/sysdata.rda -------------------------------------------------------------------------------- /neonUtilities/R/table_types.R: -------------------------------------------------------------------------------- 1 | #' Publication table information 2 | #' 3 | #' A dataset containing publication table names, descriptions, type (site-date, site-all, lab-all, lab-current), and a time index 4 | #' 5 | #' @format A data frame with 5 variables. Number of rows changes frequently as more tables are added: 6 | #' \describe{ 7 | #' \item{productID}{data product ID} 8 | #' \item{tableName}{name of table} 9 | #' \item{tableDesc}{description of table} 10 | #' \item{tableType}{type of table (important for knowing which tables to stack, and how to stack)} 11 | #' \item{tableTMI}{a time index (e.g., 0 = native resolution, 1 = 1 minute, 30 = 30 minute averages or totals)} 12 | #' } 13 | #' @source NEON database 14 | "table_types" 15 | -------------------------------------------------------------------------------- /neonUtilities/R/timeStampSet.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Generate a consensus set of time stamps from a set of input tables. 3 | 4 | #' @author 5 | #' Claire Lunch \email{clunch@battelleecology.org} 6 | 7 | #' @description 8 | #' Generate consensus SAE time stamps from a set of tables. Used in stackEddy(), not intended for independent use. 9 | #' 10 | #' @keywords internal 11 | #' @param tabList A list of SAE data tables 12 | #' @return A table of time stamps (start and end times) aggregated from the input tables 13 | 14 | #' @references 15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 16 | 17 | # Changelog and author contributions / copyrights 18 | # Claire Lunch (2023-05-30) 19 | ############################################################################################## 20 | 21 | timeStampSet <- function(tabList){ 22 | 23 | nameSet <- c("timeBgn","timeEnd") 24 | 25 | # get a set of time stamps to initiate the table. leave out qfqm to exclude 26 | # filler records created as placeholders for days with no data 27 | timeSet <- tabList[grep("qfqm", names(tabList), invert=T)] 28 | # turbulent flux and footprint end time stamps don't quite match the others 29 | timeSet <- timeSet[grep("turb", names(timeSet), invert=T)] 30 | timeSet <- timeSet[grep("foot", names(timeSet), invert=T)] 31 | 32 | # initiate the table with consensus set of time stamps 33 | timeSetInit <- timeSet[[1]][,nameSet] 34 | if(length(timeSet)==1) { 35 | timeSetInit <- timeSetInit 36 | } else { 37 | for(q in 2:length(timeSet)) { 38 | # check for additional start time stamps 39 | timeSetTemp <- timeSet[[q]][,nameSet] 40 | timeSetTempMerg <- data.table::as.data.table(timeSetTemp[,"timeBgn"]) 41 | timeSetInitMerg <- data.table::as.data.table(timeSetInit[,"timeBgn"]) 42 | misTime <- data.table::fsetdiff(timeSetTempMerg, timeSetInitMerg) 43 | if(nrow(misTime)==0) { 44 | timeSetInit <- timeSetInit 45 | } else { 46 | # combine all, then de-dup 47 | allTime <- data.table::rbindlist(list(timeSetInit, timeSetTemp), fill=TRUE) 48 | timeSetInit <- as.data.frame(unique(allTime, by="timeBgn")) 49 | } 50 | } 51 | } 52 | 53 | return(data.table::as.data.table(timeSetInit)) 54 | 55 | } 56 | 57 | -------------------------------------------------------------------------------- /neonUtilities/R/transformFileToGeoCSV.R: -------------------------------------------------------------------------------- 1 | ############################################################################################## 2 | #' @title Transform NEON CSV file to GeoCSV 3 | #' 4 | #' @author 5 | #' Christine Laney \email{claney@battelleecology.org} 6 | 7 | #' @description 8 | #' Read in a NEON monthly data zip file and parse the respective variables file to create a new GeoCSV file 9 | #' 10 | #' @param infile The path to the file that needs to be parsed 11 | #' @param varfile The path to the variables file needed to parse the infile 12 | #' @param outfile The path where the new GeoCSV file should be placed 13 | #' @return The same data file with a GeoCSV header 14 | 15 | #' @references 16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | 18 | #' @export 19 | 20 | # Changelog and author contributions / copyrights 21 | # 2018-01-09 (Christine Laney): Created function 22 | # 2018-04-04 (Christine Laney): Updated function documentation 23 | # 2018-04-17 (Christine Laney): Added field descriptions under the keyword field_long_name 24 | ############################################################################################## 25 | 26 | transformFileToGeoCSV <- function(infile, varfile, outfile){ 27 | datafile <- utils::read.csv(infile, header = TRUE, stringsAsFactors = FALSE) 28 | vars <- utils::read.csv(varfile, header = TRUE, stringsAsFactors = FALSE) 29 | dfsplit <- strsplit(infile, "\\/") 30 | dfname <- dfsplit[[1]][length(dfsplit[[1]])] 31 | 32 | line_dataset <- "# dataset: GeoCSV 2.0\n" 33 | line_title <- paste0("# title: ", getTitle(dfname)) 34 | line_institution <- "# institution: National Ecological Observatory Network (NEON)" 35 | line_attribution <- "# attribution: http://www.neonscience.org/data-resources/data-usage-citations" 36 | line_resource <- "# resource: http://data.neonscience.org" 37 | line_field_long_name <- "# field_long_name: " 38 | line_field_unit <- "# field_unit: " 39 | line_field_type <- "# field_type: " 40 | line_delimiter <- '# delimiter: ","' 41 | for(i in 1:ncol(datafile)){ 42 | type <- vars$dataType[which(names(datafile)[i] == vars$fieldName)][1] 43 | if(is.na(type)){type = "NA"} 44 | if(type == "dateTime"){unit = "ISO_8601"} 45 | if(type %in% c("unsigned integer","signed integer")){type = "integer"} 46 | if(type == "real"){type = "float"} 47 | if(!(type %in% c("ISO_8601","integer","float"))){type = "string"} 48 | 49 | unit <- vars$units[which(names(datafile)[i] == vars$fieldName)][1] 50 | if(is.na(unit) || unit == "NA"){unit = "unitless"} 51 | 52 | long_name <- vars$description[which(names(datafile)[i] == vars$fieldName)][1] 53 | 54 | if(i == 1){ 55 | line_field_long_name <- paste0(line_field_long_name, long_name) 56 | line_field_unit <- paste0(line_field_unit, unit) 57 | line_field_type <- paste0(line_field_type, type) 58 | } 59 | if(i > 1){ 60 | line_field_long_name <- paste(line_field_long_name, long_name, sep = ", ") 61 | line_field_type <- paste(line_field_type, type, sep = ", ") 62 | line_field_unit <- paste(line_field_unit, unit, sep = ", ") 63 | } 64 | } 65 | 66 | addText <- function(object, outfile, sep = "\n", append = TRUE){ 67 | cat(object, file = outfile, sep = sep, append = append) 68 | } 69 | addText(line_dataset, outfile, append = FALSE) 70 | addText(line_title, outfile) 71 | addText(line_institution, outfile) 72 | addText(line_attribution, outfile) 73 | addText(line_resource, outfile) 74 | addText(line_field_long_name, outfile) 75 | addText(line_field_type, outfile) 76 | addText(line_field_unit, outfile) 77 | utils::write.table(datafile, file = outfile, row.names = FALSE, append = TRUE, sep = ",", 78 | fileEncoding = "UTF-8", quote = FALSE) 79 | } 80 | 81 | 82 | -------------------------------------------------------------------------------- /neonUtilities/data-raw/update_table_types.R: -------------------------------------------------------------------------------- 1 | # update_table_types.R 2 | ############################################################################################## 3 | #' @title Update table_types data frame in sysdata.rda 4 | 5 | #' @author 6 | #' Christine Laney \email{claney@battelleecology.org} 7 | 8 | #' @description 9 | #' Connect to the DPS database, pull in updated information about tables, and save locally. 10 | 11 | #' @return A saved .rda file 12 | 13 | #' @references 14 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 15 | 16 | # Changelog and author contributions / copyrights 17 | # Claire Lunch (2018-10-05) converted to get tables from PDR via the data service 18 | # Christine Laney (2017-10-19) 19 | ############################################################################################## 20 | 21 | update_table_types <- function(){ 22 | 23 | options(stringsAsFactors=F) 24 | 25 | # get publication tables from PDR 26 | req <- httr::GET(Sys.getenv("PUB_TABLES")) 27 | rawx <- XML::xmlToList(httr::content(req, as="text")) 28 | 29 | ids <- substring(unlist(lapply(rawx, '[', "dataProductId")), 15, 27) 30 | tables <- unlist(lapply(rawx, '[', "tableName")) 31 | tables <- gsub("_pub", "", tables, fixed=T) 32 | descs <- unlist(lapply(rawx, '[', "description")) 33 | typs <- unlist(lapply(rawx, '[', "tableType")) 34 | temp <- unlist(lapply(rawx, '[', "pubField")) 35 | tmi <- temp[grep("timeIndex", names(temp))] 36 | 37 | table_types <- data.frame(cbind(ids[-length(ids)], tables[-length(tables)], 38 | descs[-length(descs)], typs[-length(typs)], tmi)) 39 | colnames(table_types) <- c("productID", "tableName", "tableDesc", 40 | "tableType", "tableTMI") 41 | 42 | # need to keep table definitions for tables that only exist in RELEASE-2021 43 | table_types <- rbind(table_types, release_2021) 44 | 45 | rownames(table_types) <- 1:nrow(table_types) 46 | 47 | # term definitions for fields added by stackByTable 48 | added_fields <- data.frame(cbind(fieldName=c('domainID','siteID','horizontalPosition', 49 | 'verticalPosition','publicationDate', 50 | 'release'), 51 | description=c('Unique identifier of the NEON domain', 52 | 'NEON site code', 53 | 'Index of horizontal location at a NEON site', 54 | 'Index of vertical location at a NEON site', 55 | 'Date of data publication on the NEON data portal', 56 | 'Identifier for data release'), 57 | dataType=c(rep('string',4),'dateTime','string'), 58 | units=rep(NA,6), 59 | downloadPkg=rep('appended by stackByTable',6), 60 | pubFormat=rep(NA,6), 61 | primaryKey=rep('N',6), 62 | categoricalCodeName=rep('',6))) 63 | 64 | usethis::use_data(table_types, added_fields, science_review_variables, shared_flights, 65 | shared_aquatic, release_2021, chem_bundles, other_bundles, relevant_EPSG, 66 | cit_prov_template, internal=TRUE, overwrite=TRUE) 67 | usethis::use_data(table_types, shared_flights, shared_aquatic, chem_bundles, other_bundles, 68 | internal=FALSE, overwrite=TRUE) 69 | 70 | # write tables to repo for use outside package 71 | data.table::fwrite(added_fields, paste(Sys.getenv('NU_REPO'), 'helper_files/added_fields.csv', sep='/')) 72 | data.table::fwrite(chem_bundles, paste(Sys.getenv('NU_REPO'), 'helper_files/chem_bundles.csv', sep='/')) 73 | data.table::fwrite(other_bundles, paste(Sys.getenv('NU_REPO'), 'helper_files/other_bundles.csv', sep='/')) 74 | data.table::fwrite(release_2021, paste(Sys.getenv('NU_REPO'), 'helper_files/release_2021.csv', sep='/')) 75 | data.table::fwrite(shared_aquatic, paste(Sys.getenv('NU_REPO'), 'helper_files/shared_aquatic.csv', sep='/')) 76 | data.table::fwrite(shared_flights, paste(Sys.getenv('NU_REPO'), 'helper_files/shared_flights.csv', sep='/')) 77 | data.table::fwrite(table_types, paste(Sys.getenv('NU_REPO'), 'helper_files/table_types.csv', sep='/')) 78 | 79 | } 80 | 81 | update_table_types() 82 | -------------------------------------------------------------------------------- /neonUtilities/data/chem_bundles.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/chem_bundles.rda -------------------------------------------------------------------------------- /neonUtilities/data/other_bundles.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/other_bundles.rda -------------------------------------------------------------------------------- /neonUtilities/data/shared_aquatic.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/shared_aquatic.rda -------------------------------------------------------------------------------- /neonUtilities/data/shared_flights.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/shared_flights.rda -------------------------------------------------------------------------------- /neonUtilities/data/table_types.rda: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/table_types.rda -------------------------------------------------------------------------------- /neonUtilities/inst/extdata/NEON_gp.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/inst/extdata/NEON_gp.zip -------------------------------------------------------------------------------- /neonUtilities/inst/extdata/expected/gp/issueLog_10109.csv: -------------------------------------------------------------------------------- 1 | "id","parentIssueID","issueDate","resolvedDate","dateRangeStart","dateRangeEnd","locationAffected","issue","resolution" 2 | 16095,NA,"2020-06-01T00:00:00Z","2020-06-01T00:00:00Z","2018-11-01T00:00:00Z","2020-06-01T00:00:00Z","All terrestrial sites","After consultation with external working group community members and the NSF, The Soil Microbe Group Abundances data product was discontinued in 2020, with the last field samples having associated group abundance data in 10-2018. Users who are interested in generating this data from NEON samples may submit a request for samples or DNA extracts from the NEON Biorepository. Alternatively, the Soil Microbe Biomass data product (DP1.10104) may be utilized as a similar measurement of the quantities of microbes in soils.","Data Product discontinued" 3 | -------------------------------------------------------------------------------- /neonUtilities/inst/extdata/expected/gp/mga_labSummary.csv: -------------------------------------------------------------------------------- 1 | uid,targetTaxonGroup,labSpecificStartDate,labSpecificEndDate,laboratoryName,qpcrDetectionMethod,primerConcentration,forwardPrimer,reversePrimer,ampliconLength,dNTPConcentration,probeConcentration,mgConcentration,polymeraseType,polymeraseUnitNumber,bufferID,bufferComposition,reactionVolume,qpcrMethod,instrument,primerSpecificity,detectionLimit,multiplexStatus,qPCRAnalysisProgram,cqMethod,recordedBy,dataQF,publicationDate 2 | 91969727-5313-4707-b6d4-2aaeecaa01ab,fungi,2017-12-08,2018-02-25,Battelle Applied Genomics,DNA binding dye,0.2,CTTGGTCATTTAGAGGAAGTAA,GCTGCGTTCTTCATCGATGC,325,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v3,ABI 7500 Fast,,4,N,ABI 7500 v2.3,BMI_qPCR_SOP_v3,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z 3 | fccbb845-2684-486c-b44f-a47fd27713dc,bacteria and archaea,2017-12-08,2018-02-25,Battelle Applied Genomics,DNA binding dye,0.2,CCTACGGGNBGCASCAG,GGACTACNVGGGTATCTAATCC,467,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v3,ABI 7500 Fast,,22,N,ABI 7500 v2.3,BMI_qPCR_SOP_v3,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z 4 | 49cb98f9-8277-4d39-87bc-316ff97637b8,bacteria and archaea,2017-12-08,2018-02-25,Battelle Applied Genomics,DNA binding dye,0.2,CCTACGGGNBGCASCAG,GGACTACNVGGGTATCTAATCC,467,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v3,ABI 7500 Fast,,22,N,ABI 7500 v2.3,BMI_qPCR_SOP_v2,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z 5 | 3f96ee99-887b-48d9-bcc6-0485cf67e24a,fungi,2017-12-08,2018-02-25,Battelle Applied Genomics,DNA binding dye,0.2,CTTGGTCATTTAGAGGAAGTAA,GCTGCGTTCTTCATCGATGC,325,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v3,ABI 7500 Fast,,4,N,ABI 7500 v2.3,BMI_qPCR_SOP_v2,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z 6 | ff9c7fdb-06db-429a-bcb8-58f9e3f63122,bacteria and archaea,2017-09-01,2017-11-14,Battelle Applied Genomics,DNA binding dye,0.2,CCTACGGGNBGCASCAG,GGACTACNVGGGTATCTAATCC,467,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v1,ABI 7500 Fast,,22,N,ABI 7500 v2.3,BMI_qPCR_SOP_v1,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z 7 | 3fb531f3-f932-449f-b8da-6e239c9828bc,fungi,2017-09-01,2017-11-14,Battelle Applied Genomics,DNA binding dye,0.2,CTTGGTCATTTAGAGGAAGTAA,GCTGCGTTCTTCATCGATGC,325,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v1,ABI 7500 Fast,,4,N,ABI 7500 v2.3,BMI_qPCR_SOP_v1,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z 8 | 67adffbb-3049-425b-aee0-6cae37e7e9d8,bacteria and archaea,2017-11-15,2017-12-08,Battelle Applied Genomics,DNA binding dye,0.2,CCTACGGGNBGCASCAG,GGACTACNVGGGTATCTAATCC,467,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v2,ABI 7500 Fast,,22,N,ABI 7500 v2.3,BMI_qPCR_SOP_v2,TuIbGgXF7vfkyq5thAKFMA==,,20180705T173721Z 9 | 9c91ab5b-ff3a-44fa-86e2-72e29c5748b3,fungi,2017-11-15,2017-12-08,Battelle Applied Genomics,DNA binding dye,0.2,CTTGGTCATTTAGAGGAAGTAA,GCTGCGTTCTTCATCGATGC,325,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v2,ABI 7500 Fast,,4,N,ABI 7500 v2.3,BMI_qPCR_SOP_v2,TuIbGgXF7vfkyq5thAKFMA==,,20180705T173721Z 10 | -------------------------------------------------------------------------------- /neonUtilities/man/assignClasses.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/assignClasses.R 3 | \name{assignClasses} 4 | \alias{assignClasses} 5 | \title{Assign correct column classes} 6 | \usage{ 7 | assignClasses(dt, inVars) 8 | } 9 | \arguments{ 10 | \item{dt}{A data frame} 11 | 12 | \item{inVars}{The variables expected in the df} 13 | } 14 | \value{ 15 | A data frame with corrected column classes 16 | } 17 | \description{ 18 | Use the variables file to assign classes to each column in each data file 19 | } 20 | \references{ 21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | } 23 | \author{ 24 | Christine Laney \email{claney@battelleecology.org} 25 | } 26 | \keyword{internal} 27 | -------------------------------------------------------------------------------- /neonUtilities/man/byEventSIM.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/byEventSIM.R 3 | \name{byEventSIM} 4 | \alias{byEventSIM} 5 | \title{Get site management data by event type.} 6 | \usage{ 7 | byEventSIM( 8 | eventType, 9 | site = "all", 10 | startdate = NA, 11 | enddate = NA, 12 | release = "current", 13 | include.provisional = FALSE, 14 | token = NA_character_ 15 | ) 16 | } 17 | \arguments{ 18 | \item{eventType}{The value of eventType to search for. Can be multiple values. See categoricalCodes file for DP1.10111.001 for possible values.} 19 | 20 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.} 21 | 22 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 23 | 24 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 25 | 26 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.} 27 | 28 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.} 29 | 30 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 31 | } 32 | \value{ 33 | A data frame of sim_eventData data, matching the query criteria. Note that metadata are not included in the data returned via this function. 34 | } 35 | \description{ 36 | Query site management data to return records matching a specific eventType. 37 | } 38 | \examples{ 39 | \dontrun{ 40 | # Search for fires across all NEON event data 41 | sim.fires <- byEventSIM(eventType="fire") 42 | 43 | # Search for grazing events at several sites 44 | sim.graz <- byEventSIM(eventType="grazing", site=c("CPER","KONA","MOAB","STER","LAJA")) 45 | } 46 | 47 | } 48 | \references{ 49 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 50 | } 51 | \author{ 52 | Claire Lunch \email{clunch@battelleecology.org} 53 | } 54 | -------------------------------------------------------------------------------- /neonUtilities/man/byFileAOP.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/byFileAOP.R 3 | \name{byFileAOP} 4 | \alias{byFileAOP} 5 | \title{Serially download all AOP files for a given site, year, and product} 6 | \usage{ 7 | byFileAOP( 8 | dpID, 9 | site, 10 | year, 11 | include.provisional = FALSE, 12 | check.size = TRUE, 13 | savepath = NA, 14 | token = NA_character_ 15 | ) 16 | } 17 | \arguments{ 18 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001} 19 | 20 | \item{site}{The four-letter code of a single NEON site, e.g. 'CLBJ'.} 21 | 22 | \item{year}{The four-digit year to search for data. Defaults to 2017.} 23 | 24 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.} 25 | 26 | \item{check.size}{T or F, should the user approve the total file size before downloading? Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.} 27 | 28 | \item{savepath}{The file path to download to. Defaults to NA, in which case the working directory is used.} 29 | 30 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 31 | } 32 | \value{ 33 | A folder in the working directory, containing all files meeting query criteria. 34 | } 35 | \description{ 36 | Query the API for AOP data by site, year, and product, and download all files found, preserving original 37 | folder structure. Downloads serially to avoid overload; may take a very long time. 38 | } 39 | \examples{ 40 | \dontrun{ 41 | # To download 2017 vegetation index data from San Joaquin Experimental Range: 42 | byFileAOP(dpID="DP3.30026.001", site="SJER", year="2017") 43 | } 44 | } 45 | \references{ 46 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 47 | } 48 | \author{ 49 | Claire Lunch \email{clunch@battelleecology.org} 50 | Christine Laney \email{claney@battelleecology.org} 51 | } 52 | -------------------------------------------------------------------------------- /neonUtilities/man/byTileAOP.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/byTileAOP.R 3 | \name{byTileAOP} 4 | \alias{byTileAOP} 5 | \title{Download AOP tiles overlapping specified coordinates for a given site, year, and product} 6 | \usage{ 7 | byTileAOP( 8 | dpID, 9 | site, 10 | year, 11 | easting, 12 | northing, 13 | buffer = 0, 14 | include.provisional = FALSE, 15 | check.size = TRUE, 16 | savepath = NA, 17 | token = NA_character_ 18 | ) 19 | } 20 | \arguments{ 21 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001} 22 | 23 | \item{site}{The four-letter code of a single NEON site, e.g. 'CLBJ'.} 24 | 25 | \item{year}{The four-digit year to search for data. Defaults to 2017.} 26 | 27 | \item{easting}{A vector containing the easting UTM coordinates of the locations to download.} 28 | 29 | \item{northing}{A vector containing the northing UTM coordinates of the locations to download.} 30 | 31 | \item{buffer}{Size, in meters, of the buffer to be included around the coordinates when determining which tiles to download. Defaults to 0. If easting and northing coordinates are the centroids of NEON TOS plots, use buffer=20.} 32 | 33 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.} 34 | 35 | \item{check.size}{T or F, should the user approve the total file size before downloading? Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.} 36 | 37 | \item{savepath}{The file path to download to. Defaults to NA, in which case the working directory is used.} 38 | 39 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 40 | } 41 | \value{ 42 | A folder in the working directory, containing all files meeting query criteria. 43 | } 44 | \description{ 45 | Query the API for AOP data by site, year, product, and tile location, and download all files found. 46 | Downloads serially to avoid overload; may take a very long time. 47 | } 48 | \references{ 49 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 50 | } 51 | \author{ 52 | Claire Lunch \email{clunch@battelleecology.org} 53 | Christine Laney \email{claney@battelleecology.org} 54 | } 55 | -------------------------------------------------------------------------------- /neonUtilities/man/chem_bundles.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/chem_bundles.R 3 | \docType{data} 4 | \name{chem_bundles} 5 | \alias{chem_bundles} 6 | \title{Bundled chemistry data product information} 7 | \format{ 8 | A data frame with 2 variables: 9 | \describe{ 10 | \item{product}{Data product ID of a terrestrial chemistry product} 11 | \item{homeProduct}{Data product ID of the corresponding home data product} 12 | } 13 | } 14 | \source{ 15 | NEON data product bundles 16 | } 17 | \usage{ 18 | chem_bundles 19 | } 20 | \description{ 21 | A dataset containing NEON data product codes of terrestrial chemistry data products and the "home" data products they are bundled with. 22 | } 23 | \keyword{datasets} 24 | -------------------------------------------------------------------------------- /neonUtilities/man/cleanUp.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/cleanUp.R 3 | \name{cleanUp} 4 | \alias{cleanUp} 5 | \title{Clean up folder after stacking} 6 | \usage{ 7 | cleanUp(folder, orig) 8 | } 9 | \arguments{ 10 | \item{folder}{The file path to the folder that needs to be cleaned up (the root directory of the data package)} 11 | 12 | \item{orig}{The list of files that were present in the folder before unzipping and stacking} 13 | } 14 | \value{ 15 | Only the folders created during unzip will be deleted. All custom folders/files and the stackedFiles output folder will be retained. 16 | } 17 | \description{ 18 | Remove unzipped monthly data folders 19 | } 20 | \references{ 21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | } 23 | \author{ 24 | Christine Laney \email{claney@battelleecology.org} 25 | } 26 | \keyword{internal} 27 | -------------------------------------------------------------------------------- /neonUtilities/man/convByteSize.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/convByteSize.R 3 | \name{convByteSize} 4 | \alias{convByteSize} 5 | \title{Convert a number of bytes into megabytes or gigabytes} 6 | \usage{ 7 | convByteSize(objSize) 8 | } 9 | \arguments{ 10 | \item{objSize}{The size in bytes} 11 | } 12 | \value{ 13 | The size of the file in megabytes or gigabytes 14 | } 15 | \description{ 16 | For any number of bytes, convert to a number of MB or GB 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Claire Lunch \email{clunch@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/datasetQuery.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/datasetQuery.R 3 | \name{datasetQuery} 4 | \alias{datasetQuery} 5 | \title{Query the query endpoint of the NEON API and create an arrow dataset from the results} 6 | \usage{ 7 | datasetQuery( 8 | dpID, 9 | site = "all", 10 | startdate = NA, 11 | enddate = NA, 12 | tabl = NA, 13 | hor = NA, 14 | ver = NA, 15 | package = "basic", 16 | release = "current", 17 | include.provisional = FALSE, 18 | token = NA_character_ 19 | ) 20 | } 21 | \arguments{ 22 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001} 23 | 24 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.} 25 | 26 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 27 | 28 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 29 | 30 | \item{tabl}{The name of a single data table to download.} 31 | 32 | \item{hor}{The horizontal index of data to download. Only applicable to sensor (IS) data.} 33 | 34 | \item{ver}{The vertical index of data to download. Only applicable to sensor (IS) data.} 35 | 36 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.} 37 | 38 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.} 39 | 40 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.} 41 | 42 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.} 43 | } 44 | \value{ 45 | An arrow dataset for the data requested. 46 | } 47 | \description{ 48 | Uses the query endpoint of the NEON API to find the full list of files for a given data product, release, site(s), and date range, then turns them into an arrow dataset. 49 | } 50 | \references{ 51 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 52 | } 53 | \author{ 54 | Claire Lunch \email{clunch@battelleecology.org} 55 | } 56 | -------------------------------------------------------------------------------- /neonUtilities/man/dateConvert.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/dateConvert.R 3 | \name{dateConvert} 4 | \alias{dateConvert} 5 | \title{Convert date stamps from character} 6 | \usage{ 7 | dateConvert(dates, useFasttime = FALSE) 8 | } 9 | \arguments{ 10 | \item{dates}{A vector of date values in character format [character]} 11 | 12 | \item{useFasttime}{Should the fasttime package be used for date conversion? Defaults to false. [logical]} 13 | } 14 | \value{ 15 | A POSIXct vector, if possible; if conversion was unsuccessful, the original vector is returned 16 | } 17 | \description{ 18 | Attempt to convert date stamps from character, iterating through known NEON date formats 19 | } 20 | \references{ 21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | } 23 | \author{ 24 | Claire Lunch \email{clunch@battelleecology.org} 25 | } 26 | \keyword{internal} 27 | -------------------------------------------------------------------------------- /neonUtilities/man/eddyStampCheck.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/eddyStampCheck.R 3 | \name{eddyStampCheck} 4 | \alias{eddyStampCheck} 5 | \title{Convert date stamps from character and check for only one record in a day} 6 | \usage{ 7 | eddyStampCheck(tab, useFasttime = FALSE) 8 | } 9 | \arguments{ 10 | \item{tab}{A table of SAE data} 11 | 12 | \item{useFasttime}{Should the fasttime package be used to convert time stamps?} 13 | } 14 | \value{ 15 | The same table of SAE data, with time stamps converted and empty records representing a single day (filler records inserted during processing) removed. 16 | } 17 | \description{ 18 | Convert SAE time stamps to POSIX and check for missing data 19 | } 20 | \references{ 21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | } 23 | \author{ 24 | Claire Lunch \email{clunch@battelleecology.org} 25 | } 26 | \keyword{internal} 27 | -------------------------------------------------------------------------------- /neonUtilities/man/findDatatables.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/findDatatables.R 3 | \name{findDatatables} 4 | \alias{findDatatables} 5 | \title{Find data tables} 6 | \usage{ 7 | findDatatables(folder, fnames = T) 8 | } 9 | \arguments{ 10 | \item{folder}{The folder of the outputs} 11 | 12 | \item{fnames}{Full names - if true, then return the full file names including enclosing folders, if false, return only the file names} 13 | } 14 | \value{ 15 | a data frame of file names 16 | } 17 | \description{ 18 | List the names of the data tables within each folder 19 | } 20 | \references{ 21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | } 23 | \author{ 24 | Christine Laney \email{claney@battelleecology.org} 25 | } 26 | \keyword{internal} 27 | -------------------------------------------------------------------------------- /neonUtilities/man/findTablesByFormat.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/findTablesByFormat.R 3 | \name{findTablesByFormat} 4 | \alias{findTablesByFormat} 5 | \title{Find unique data tables in dataset} 6 | \usage{ 7 | findTablesByFormat(datatables) 8 | } 9 | \arguments{ 10 | \item{datatables}{A list of data files} 11 | } 12 | \value{ 13 | An array of unique table names and their types 14 | } 15 | \description{ 16 | Find the unique data tables that are present in the dataset (e.g., 2 minute vs 30 minute, or pinning vs identification data) and their types, based on the file name formatting. Adapted from findTablesUnique(). 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Claire Lunch \email{clunch@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/findTablesUnique.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/findTablesUnique.R 3 | \name{findTablesUnique} 4 | \alias{findTablesUnique} 5 | \title{Find unique data tables in dataset} 6 | \usage{ 7 | findTablesUnique(datatables, tabletypes) 8 | } 9 | \arguments{ 10 | \item{datatables}{A list of data files} 11 | } 12 | \value{ 13 | An array of unique table names 14 | } 15 | \description{ 16 | Find the unique data tables that are present in the dataset (e.g., 2 minute vs 30 minute, or pinning vs identification data) 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Christine Laney \email{claney@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/footRaster.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/footRaster.R 3 | \name{footRaster} 4 | \alias{footRaster} 5 | \title{Extract eddy covariance footprint data from HDF5 format} 6 | \usage{ 7 | footRaster(filepath) 8 | } 9 | \arguments{ 10 | \item{filepath}{One of: a folder containing NEON EC H5 files, a zip file of DP4.00200.001 data downloaded from the NEON data portal, a folder of DP4.00200.001 data downloaded by the neonUtilities::zipsByProduct() function, or a single NEON EC H5 file. Filepath can only contain files for a single site. [character]} 11 | } 12 | \value{ 13 | A rasterStack object containing all the footprints in the input files, plus one layer (the first in the stack) containing the mean footprint. 14 | } 15 | \description{ 16 | Create a raster of flux footprint data. Specific to expanded package of eddy covariance data product: DP4.00200.001 17 | For definition of a footprint, see Glossary of Meteorology: https://glossary.ametsoc.org/wiki/Footprint 18 | For background information about flux footprints and considerations around the time scale of footprint calculations, see Amiro 1998: https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.922.4124&rep=rep1&type=pdf 19 | } 20 | \details{ 21 | Given a filepath containing H5 files of expanded package DP4.00200.001 data, extracts flux footprint data and creates a raster. 22 | } 23 | \examples{ 24 | \dontrun{ 25 | # To run the function on a zip file downloaded from the NEON data portal: 26 | ftprnt <- footRaster(filepath="~/NEON_eddy-flux.zip") 27 | } 28 | } 29 | \references{ 30 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 31 | } 32 | \author{ 33 | Claire Lunch \email{clunch@battelleecology.org} 34 | } 35 | -------------------------------------------------------------------------------- /neonUtilities/man/getAPI.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getAPI.R 3 | \name{getAPI} 4 | \alias{getAPI} 5 | \title{Get the data from API} 6 | \usage{ 7 | getAPI(apiURL, token = NA_character_) 8 | } 9 | \arguments{ 10 | \item{apiURL}{The API endpoint URL} 11 | 12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.} 13 | } 14 | \description{ 15 | Accesses the API with options to use the user-specific API token generated within data.neonscience.org user accounts. 16 | } 17 | \references{ 18 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 19 | } 20 | \author{ 21 | Nate Mietkiewicz \email{mietkiewicz@battelleecology.org} 22 | } 23 | \keyword{internal} 24 | -------------------------------------------------------------------------------- /neonUtilities/man/getAPIHeaders.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getAPIHeaders.R 3 | \name{getAPIHeaders} 4 | \alias{getAPIHeaders} 5 | \title{Get only headers from API} 6 | \usage{ 7 | getAPIHeaders(apiURL, token = NA_character_) 8 | } 9 | \arguments{ 10 | \item{apiURL}{The API endpoint URL} 11 | 12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.} 13 | } 14 | \description{ 15 | Accesses the API with options to use the user-specific API token generated within neon.datascience user accounts. 16 | } 17 | \references{ 18 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 19 | } 20 | \author{ 21 | Claire Lunch \email{clunch@battelleecology.org} 22 | } 23 | \keyword{internal} 24 | -------------------------------------------------------------------------------- /neonUtilities/man/getAttributes.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getAttributes.R 3 | \name{getAttributes} 4 | \alias{getAttributes} 5 | \title{Extract attributes from eddy covariance H5 files} 6 | \usage{ 7 | getAttributes(fil, sit, type, valName) 8 | } 9 | \arguments{ 10 | \item{fil}{File path to the H5 file to extract attributes from [character]} 11 | 12 | \item{sit}{The site, for site attributes. Must match site of file path. [character]} 13 | 14 | \item{type}{The type of attributes to retrieve. [character]} 15 | 16 | \item{valName}{If CO2 validation metadata are requested, the H5 name of the level where they can be found. [character]} 17 | } 18 | \value{ 19 | A data frame with one row containing the extracted attributes 20 | } 21 | \description{ 22 | Extract attribute metadata from H5 files 23 | } 24 | \references{ 25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 26 | } 27 | \author{ 28 | Claire Lunch \email{clunch@battelleecology.org} 29 | } 30 | \keyword{internal} 31 | -------------------------------------------------------------------------------- /neonUtilities/man/getAvg.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getAvg.R 3 | \name{getAvg} 4 | \alias{getAvg} 5 | \title{Get a list of the available averaging intervals for a data product} 6 | \usage{ 7 | getAvg(dpID, token = NA_character_) 8 | } 9 | \arguments{ 10 | \item{dpID}{The identifier of the NEON data product, in the form DPL.PRNUM.REV, e.g. DP1.00006.001} 11 | 12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 13 | } 14 | \value{ 15 | A vector of the available averaging intervals, typically in minutes. 16 | } 17 | \description{ 18 | Most IS products are available at multiple averaging intervals; get a list of what's available for a given data product 19 | } 20 | \examples{ 21 | # Get available averaging intervals for PAR data product 22 | getAvg("DP1.00024.001") 23 | } 24 | \references{ 25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 26 | } 27 | \author{ 28 | Claire Lunch \email{clunch@battelleecology.org} 29 | } 30 | -------------------------------------------------------------------------------- /neonUtilities/man/getCitation.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getCitation.R 3 | \name{getCitation} 4 | \alias{getCitation} 5 | \title{Get a Bibtex citation for NEON data with a DOI, or generate a provisional Bibtex citation} 6 | \usage{ 7 | getCitation(dpID = NA_character_, release = NA_character_) 8 | } 9 | \arguments{ 10 | \item{dpID}{The data product ID of the data to be cited [character]} 11 | 12 | \item{release}{The data release to be cited. Can be provisional. [character]} 13 | } 14 | \value{ 15 | A character string containing the Bibtex citation 16 | } 17 | \description{ 18 | Use the DOI Foundation API to get Bibtex-formatted citations for NEON data, or use a template to generate a Bibtex citation for provisional data. Helper function to download and stacking functions. 19 | } 20 | \examples{ 21 | \dontrun{ 22 | # Get the citation for Breeding landbird point counts (DP1.10003.001), RELEASE-2023 23 | cit <- getCitation(dpID="DP1.10003.001", release="RELEASE-2023") 24 | } 25 | } 26 | \references{ 27 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 28 | } 29 | \author{ 30 | Claire Lunch \email{clunch@battelleecology.org} 31 | } 32 | -------------------------------------------------------------------------------- /neonUtilities/man/getDatatable.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getDatatable.R 3 | \name{getDatatable} 4 | \alias{getDatatable} 5 | \title{Get NEON data table} 6 | \usage{ 7 | getDatatable( 8 | dpid = NA, 9 | data_table_name = NA, 10 | sample_location_list = NA, 11 | sample_location_type = "siteID", 12 | sample_date_min = "2012-01-01", 13 | sample_date_max = Sys.Date(), 14 | sample_date_format = "\%Y-\%m-\%d", 15 | data_package_type = "basic", 16 | url_prefix_data = "https://data.neonscience.org/api/v0/data/", 17 | url_prefix_products = "https://data.neonscience.org/api/v0/products/", 18 | token = NA_character_ 19 | ) 20 | } 21 | \arguments{ 22 | \item{dpid}{character sting for NEON data product ID} 23 | 24 | \item{data_table_name}{character sting for name of the data table to download, e.g., 'sls_soilCoreCollection'} 25 | 26 | \item{sample_location_list}{list of sites, domains, etc. If NA, retrieve all data for the given data table / dpid combination.} 27 | 28 | \item{sample_location_type}{character sting for location type, such as 'siteID'. Must be one of the NEON controlled terms. If you're unsure, use 'siteID'} 29 | 30 | \item{sample_date_min}{start date for query. Default is 1-Jan-2012, and this should capture the earliest NEON data record.} 31 | 32 | \item{sample_date_max}{end date for query. Default is current date.} 33 | 34 | \item{sample_date_format}{date format. Default/expected format is yyyy-mm-dd} 35 | 36 | \item{data_package_type}{package type, either 'basic' or 'expanded'. If unsure, use 'expanded'} 37 | 38 | \item{url_prefix_data}{data endpoint for NEON API.} 39 | 40 | \item{url_prefix_products}{products endpoint for NEON API.} 41 | 42 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 43 | } 44 | \value{ 45 | data frame with selected NEON data 46 | } 47 | \description{ 48 | This is a function to retrieve a data table 49 | from the NEON data portal for sites and dates provided by the 50 | enduser. NOTE that this function only works for NEON 51 | Observation System (OS) data products, and only for select tables 52 | } 53 | \references{ 54 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 55 | } 56 | \author{ 57 | Eric R. Sokol \email{esokol@battelleecology.org} 58 | } 59 | -------------------------------------------------------------------------------- /neonUtilities/man/getEddyLog.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getEddyLog.R 3 | \name{getEddyLog} 4 | \alias{getEddyLog} 5 | \title{Get the full issue log set for the SAE bundle} 6 | \usage{ 7 | getEddyLog(token = NA_character_) 8 | } 9 | \arguments{ 10 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 11 | } 12 | \value{ 13 | A table of issues reported for the data product. 14 | } 15 | \description{ 16 | Use the NEON API to get the issue log from all products in the bundle in a user-friendly format 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Claire Lunch \email{clunch@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/getFileUrls.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getFileUrls.R 3 | \name{getFileUrls} 4 | \alias{getFileUrls} 5 | \title{Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame} 6 | \usage{ 7 | getFileUrls(m.urls, include.provisional, token = NA) 8 | } 9 | \arguments{ 10 | \item{m.urls}{The monthly API URL for the AOP files} 11 | 12 | \item{include.provisional}{T or F, should provisional data be included in downloaded files?} 13 | 14 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 15 | } 16 | \value{ 17 | A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0) 18 | } 19 | \description{ 20 | Used to generate a data frame of available AOP files. 21 | } 22 | \references{ 23 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 24 | } 25 | \author{ 26 | Claire Lunch \email{clunch@battelleecology.org} 27 | Christine Laney \email{claney@battelleecology.org} 28 | } 29 | \keyword{internal} 30 | -------------------------------------------------------------------------------- /neonUtilities/man/getIssueLog.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getIssueLog.R 3 | \name{getIssueLog} 4 | \alias{getIssueLog} 5 | \title{Get the issue log for a specific data product} 6 | \usage{ 7 | getIssueLog(dpID = NA, token = NA_character_) 8 | } 9 | \arguments{ 10 | \item{dpID}{The data product identifier, formatted as DP#.#####.###} 11 | 12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 13 | } 14 | \value{ 15 | A table of issues reported for the data product. 16 | } 17 | \description{ 18 | Use the NEON API to get the issue log in a user-friendly format 19 | } 20 | \examples{ 21 | # Get documentation and availability of plant foliar properties data product 22 | cfcIssues <- getIssueLog("DP1.10026.001") 23 | } 24 | \references{ 25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 26 | } 27 | \author{ 28 | Claire Lunch \email{clunch@battelleecology.org} 29 | } 30 | -------------------------------------------------------------------------------- /neonUtilities/man/getNeonDOI.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getNeonDOI.R 3 | \name{getNeonDOI} 4 | \alias{getNeonDOI} 5 | \title{Get either a list of NEON DOIs, or the DOI for a specific data product and release} 6 | \usage{ 7 | getNeonDOI(dpID = NA_character_, release = NA_character_) 8 | } 9 | \arguments{ 10 | \item{dpID}{The data product identifier, formatted as DP#.#####.### [character]} 11 | 12 | \item{release}{Name of a specific release, e.g. RELEASE-2022 [character]} 13 | } 14 | \value{ 15 | A table of data product IDs and DOIs. 16 | } 17 | \description{ 18 | Use the DataCite API to get NEON data DOIs in a user-friendly format 19 | } 20 | \examples{ 21 | \dontrun{ 22 | # Get all NEON data DOIs 23 | allDOIs <- getNeonDOI() 24 | } 25 | } 26 | \references{ 27 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 28 | } 29 | \author{ 30 | Claire Lunch \email{clunch@battelleecology.org} 31 | } 32 | -------------------------------------------------------------------------------- /neonUtilities/man/getPackage.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getPackage.R 3 | \name{getPackage} 4 | \alias{getPackage} 5 | \title{Get NEON data package} 6 | \usage{ 7 | getPackage(dpID, site_code, year_month, package = "basic", savepath = getwd()) 8 | } 9 | \arguments{ 10 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001} 11 | 12 | \item{site_code}{A four-letter NEON research site code, such as HEAL for Healy.} 13 | 14 | \item{year_month}{The year and month of interest, in format YYYY-MM.} 15 | 16 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.} 17 | 18 | \item{savepath}{The location to save the output files to} 19 | } 20 | \value{ 21 | A zipped monthly file 22 | } 23 | \description{ 24 | Get a zipped file for a single data product, site, and year-month combination. Use the NEON data portal or API to determine data 25 | availability by data product, site, and year-month combinations. 26 | } 27 | \references{ 28 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 29 | } 30 | \author{ 31 | Christine Laney \email{claney@battelleecology.org} 32 | } 33 | -------------------------------------------------------------------------------- /neonUtilities/man/getProductInfo.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getProductInfo.R 3 | \name{getProductInfo} 4 | \alias{getProductInfo} 5 | \title{Get NEON data product information} 6 | \usage{ 7 | getProductInfo(dpID = "", token = NA) 8 | } 9 | \arguments{ 10 | \item{dpID}{The data product id (optional), formated as DP#.#####.###} 11 | 12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 13 | } 14 | \value{ 15 | A named list of metadata and availability information for a single data product. If the dpID argument is omitted, a table of information for all data products in the NEON catalog. 16 | } 17 | \description{ 18 | Use the NEON API to get data product information such as availability, science team, etc. 19 | } 20 | \examples{ 21 | # Get documentation and availability of plant foliar properties data product 22 | cfcInfo <- getProductInfo("DP1.10026.001") 23 | } 24 | \references{ 25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 26 | } 27 | \author{ 28 | Christine Laney \email{claney@battelleecology.org} 29 | } 30 | -------------------------------------------------------------------------------- /neonUtilities/man/getProductSensors.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getProductSensors.R 3 | \name{getProductSensors} 4 | \alias{getProductSensors} 5 | \title{Get data product-sensor relationships} 6 | \usage{ 7 | getProductSensors() 8 | } 9 | \value{ 10 | A data frame 11 | } 12 | \description{ 13 | Pull all data from the NEON API /products endpoint, create a data frame with data product ID, data product name, and sensor type. 14 | } 15 | \examples{ 16 | \dontrun{ 17 | sensors <- getProductSensors() 18 | } 19 | } 20 | \references{ 21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | } 23 | \author{ 24 | Christine Laney \email{claney@battelleecology.org} 25 | } 26 | -------------------------------------------------------------------------------- /neonUtilities/man/getReadmePublicationDate.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getReadmePublicationDate.R 3 | \name{getReadmePublicationDate} 4 | \alias{getReadmePublicationDate} 5 | \title{Scrape the publication date from each ReadMe file} 6 | \usage{ 7 | getReadmePublicationDate(savepath, out_filepath, dpID) 8 | } 9 | \arguments{ 10 | \item{savepath}{The root folder directory where the ReadMe files are located.} 11 | 12 | \item{out_filepath}{The output directory and filename.} 13 | 14 | \item{dpID}{The data product identifier} 15 | } 16 | \description{ 17 | Given a directory, this will recursively list all of the ReadMe files that were unzipped. 18 | This should result in a single text file with a list of all of the publication dates from the ReadMe file. 19 | } 20 | \references{ 21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 22 | } 23 | \author{ 24 | Nathan Mietkiewicz \email{mietkiewicz@battelleecology.org} 25 | } 26 | \keyword{internal} 27 | -------------------------------------------------------------------------------- /neonUtilities/man/getRecentPublication.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getRecentPublication.R 3 | \name{getRecentPublication} 4 | \alias{getRecentPublication} 5 | \title{Returns the most recent files for those that do not need stacking} 6 | \usage{ 7 | getRecentPublication(inList) 8 | } 9 | \arguments{ 10 | \item{inList}{The list of files.} 11 | } 12 | \description{ 13 | Given a list of files, this will order and return the file with the most recent publication date. 14 | } 15 | \references{ 16 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 17 | } 18 | \author{ 19 | Nathan Mietkiewicz \email{mietkiewicz@battelleecology.org} 20 | } 21 | \keyword{internal} 22 | -------------------------------------------------------------------------------- /neonUtilities/man/getTaxonTable.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getTaxonTable.R 3 | \name{getTaxonTable} 4 | \alias{getTaxonTable} 5 | \title{Get NEON taxon table} 6 | \usage{ 7 | getTaxonTable( 8 | taxonType = NA, 9 | recordReturnLimit = NA, 10 | stream = "true", 11 | token = NA 12 | ) 13 | } 14 | \arguments{ 15 | \item{taxonType}{Character string for the taxonTypeCode. Must be one of ALGAE, BEETLE, BIRD, 16 | FISH, HERPETOLOGY, MACROINVERTEBRATE, MOSQUITO, MOSQUITO_PATHOGENS, SMALL_MAMMAL, PLANT, TICK} 17 | 18 | \item{recordReturnLimit}{Integer. The number of items to limit the result set to. If NA, will return all records in table.} 19 | 20 | \item{stream}{Character string, true or false. Option to obtain the result as a stream. Utilize for large requests.} 21 | 22 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 23 | } 24 | \value{ 25 | data frame with selected NEON data 26 | } 27 | \description{ 28 | This is a function to retrieve a taxon table 29 | from the NEON data portal for the taxon type by the 30 | enduser. 31 | } 32 | \references{ 33 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 34 | } 35 | \author{ 36 | Eric R. Sokol \email{esokol@battelleecology.org} 37 | } 38 | -------------------------------------------------------------------------------- /neonUtilities/man/getTileUrls.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getTileUrls.R 3 | \name{getTileUrls} 4 | \alias{getTileUrls} 5 | \title{Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame} 6 | \usage{ 7 | getTileUrls( 8 | m.urls, 9 | tileEasting, 10 | tileNorthing, 11 | include.provisional, 12 | token = NA_character_ 13 | ) 14 | } 15 | \arguments{ 16 | \item{m.urls}{The monthly API URL for the AOP tile.} 17 | 18 | \item{tileEasting}{A vector containing the easting UTM coordinates of the locations to download.} 19 | 20 | \item{tileNorthing}{A vector containing the northing UTM coordinates of the locations to download.} 21 | 22 | \item{include.provisional}{T or F, should provisional data be included in downloaded files?} 23 | 24 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.} 25 | } 26 | \value{ 27 | A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0) 28 | } 29 | \description{ 30 | Produces a data frame that is populated by available tiles for the AOP product. 31 | } 32 | \references{ 33 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 34 | } 35 | \author{ 36 | Claire Lunch \email{clunch@battelleecology.org} 37 | Christine Laney \email{claney@battelleecology.org} 38 | } 39 | \keyword{internal} 40 | -------------------------------------------------------------------------------- /neonUtilities/man/getTimeIndex.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getTimeIndex.R 3 | \name{getTimeIndex} 4 | \alias{getTimeIndex} 5 | \title{Get a list of the available time intervals for a data product} 6 | \usage{ 7 | getTimeIndex(dpID, token = NA_character_) 8 | } 9 | \arguments{ 10 | \item{dpID}{The identifier of the NEON data product, in the form DPL.PRNUM.REV, e.g. DP1.00006.001} 11 | 12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 13 | } 14 | \value{ 15 | A vector of the available time intervals, typically in minutes. 16 | } 17 | \description{ 18 | Most IS products are available at multiple time intervals; get a list of what's available for a given data product 19 | } 20 | \examples{ 21 | # Get available time intervals for PAR data product 22 | getTimeIndex("DP1.00024.001") 23 | } 24 | \references{ 25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 26 | } 27 | \author{ 28 | Claire Lunch \email{clunch@battelleecology.org} 29 | } 30 | -------------------------------------------------------------------------------- /neonUtilities/man/getTitle.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getTitle.R 3 | \name{getTitle} 4 | \alias{getTitle} 5 | \title{Get NEON data product title} 6 | \usage{ 7 | getTitle(filename) 8 | } 9 | \arguments{ 10 | \item{filename}{A NEON file name} 11 | } 12 | \value{ 13 | A title for the respective GeoCSV file 14 | } 15 | \description{ 16 | Create a title for a NEON data CSV file 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Christine Laney \email{claney@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/getVariables.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getVariables.R 3 | \name{getVariables} 4 | \alias{getVariables} 5 | \title{Get correct data types} 6 | \usage{ 7 | getVariables(varFile) 8 | } 9 | \arguments{ 10 | \item{varFile}{A file that contains variable definitions} 11 | } 12 | \value{ 13 | A data frame with fieldName and assigned column class, along with table if present 14 | } 15 | \description{ 16 | Support way to force R to read assign correct data types to each column based on variables file 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Christine Laney \email{claney@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/getVariablesEddy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getVariablesEddy.R 3 | \name{getVariablesEddy} 4 | \alias{getVariablesEddy} 5 | \title{Get variable names and units from SAE H5 files} 6 | \usage{ 7 | getVariablesEddy(tabList) 8 | } 9 | \arguments{ 10 | \item{tabList}{A list of SAE data tables} 11 | } 12 | \value{ 13 | A table of variable names and units, aggregated from the input tables 14 | } 15 | \description{ 16 | Extract variable names and units from SAE H5 files and return in user-friendly form. Used in stackEddy(), not intended for independent use. 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Claire Lunch \email{clunch@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/getVarsEddy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getVarsEddy.R 3 | \name{getVarsEddy} 4 | \alias{getVarsEddy} 5 | \title{Extract list of eddy covariance tables from HDF5 files} 6 | \usage{ 7 | getVarsEddy(filepath) 8 | } 9 | \arguments{ 10 | \item{filepath}{The folder containing the H5 file [character]} 11 | } 12 | \value{ 13 | A data frame of the metadata for each data table in the HDF5 file 14 | } 15 | \description{ 16 | Extracts a list of table metadata from a single HDF5 file. Specific to eddy covariance data product: DP4.00200.001. Can inform inputs to stackEddy(); variables listed in 'name' are available inputs to the 'var' parameter in stackEddy(). 17 | } 18 | \examples{ 19 | \dontrun{ 20 | # read variables from a file in a hypothetical filepath 21 | ec.vars <- getVarsEddy(filepath='/data/NEON.D19.BONA.DP4.00200.001.nsae.2017-12.basic.h5') 22 | } 23 | } 24 | \references{ 25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 26 | } 27 | \author{ 28 | Claire Lunch \email{clunch@battelleecology.org} 29 | } 30 | -------------------------------------------------------------------------------- /neonUtilities/man/getZipUrls.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/getZipUrls.R 3 | \name{getZipUrls} 4 | \alias{getZipUrls} 5 | \title{Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame} 6 | \usage{ 7 | getZipUrls( 8 | month.urls, 9 | avg, 10 | package, 11 | dpID, 12 | release, 13 | tabl, 14 | include.provisional, 15 | token = NA_character_ 16 | ) 17 | } 18 | \arguments{ 19 | \item{month.urls}{The monthly API URL for the URL files} 20 | 21 | \item{avg}{Global variable for averaging interval} 22 | 23 | \item{package}{Global varaible for package type (basic or expanded)} 24 | 25 | \item{dpID}{Global variable for data product ID} 26 | 27 | \item{release}{Data release to be downloaded} 28 | 29 | \item{tabl}{Table name to get} 30 | 31 | \item{include.provisional}{Should provisional data be included?} 32 | 33 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 34 | } 35 | \value{ 36 | A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0) 37 | } 38 | \description{ 39 | Used to generate a data frame of available zipfile URLs. 40 | } 41 | \references{ 42 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 43 | } 44 | \author{ 45 | Claire Lunch \email{clunch@battelleecology.org} 46 | Christine Laney \email{claney@battelleecology.org} 47 | } 48 | \keyword{internal} 49 | -------------------------------------------------------------------------------- /neonUtilities/man/listFilesInZip.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/listFilesInZip.R 3 | \name{listFilesInZip} 4 | \alias{listFilesInZip} 5 | \title{Get a data frame with the names of all files within a zipped NEON data package} 6 | \usage{ 7 | listFilesInZip(zippath) 8 | } 9 | \arguments{ 10 | \item{zippath}{The path to a zip file} 11 | } 12 | \value{ 13 | A list of filenames within the given zip file 14 | } 15 | \description{ 16 | Given the top level zip file, return dataframe of all of the files within it without 17 | unzipping the file 18 | } 19 | \references{ 20 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 21 | } 22 | \author{ 23 | Christine Laney \email{claney@battelleecology.org} 24 | } 25 | \keyword{internal} 26 | -------------------------------------------------------------------------------- /neonUtilities/man/listZipfiles.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/listZipfiles.R 3 | \name{listZipfiles} 4 | \alias{listZipfiles} 5 | \title{Get all zip file names within a zipped NEON data package} 6 | \usage{ 7 | listZipfiles(zippath) 8 | } 9 | \arguments{ 10 | \item{zippath}{The path to a zip file} 11 | } 12 | \value{ 13 | An array of all zip files contained within the focal zip file 14 | } 15 | \description{ 16 | Given the data frame of all the files within the top level zip file, 17 | return an array of just the zip file names (no pdf, xml, or other files). 18 | } 19 | \references{ 20 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 21 | } 22 | \author{ 23 | Christine Laney \email{claney@battelleecology.org} 24 | } 25 | \keyword{internal} 26 | -------------------------------------------------------------------------------- /neonUtilities/man/loadByProduct.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/loadByProduct.R 3 | \name{loadByProduct} 4 | \alias{loadByProduct} 5 | \title{Get files from NEON API, stack tables, and load into the current environment} 6 | \usage{ 7 | loadByProduct( 8 | dpID, 9 | site = "all", 10 | startdate = NA, 11 | enddate = NA, 12 | package = "basic", 13 | release = "current", 14 | timeIndex = "all", 15 | tabl = "all", 16 | check.size = TRUE, 17 | include.provisional = FALSE, 18 | nCores = 1, 19 | forceParallel = FALSE, 20 | token = NA_character_, 21 | useFasttime = FALSE, 22 | avg = NA 23 | ) 24 | } 25 | \arguments{ 26 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001} 27 | 28 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.} 29 | 30 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 31 | 32 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 33 | 34 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.} 35 | 36 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.} 37 | 38 | \item{timeIndex}{Either the string 'all', or the time index of data to download, in minutes. Only applicable to sensor (IS) data. Defaults to 'all'.} 39 | 40 | \item{tabl}{Either the string 'all', or the name of a single data table to download. Defaults to 'all'.} 41 | 42 | \item{check.size}{T or F, should the user approve the total file size before downloading? Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.} 43 | 44 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.} 45 | 46 | \item{nCores}{The number of cores to parallelize the stacking procedure. By default it is set to a single core.} 47 | 48 | \item{forceParallel}{If the data volume to be processed does not meet minimum requirements to run in parallel, this overrides. Set to FALSE as default.} 49 | 50 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)} 51 | 52 | \item{useFasttime}{Should the fasttime package be used to read date-time fields? Defaults to false.} 53 | 54 | \item{avg}{Deprecated; use timeIndex} 55 | } 56 | \value{ 57 | A named list of all the data tables in the data product downloaded, plus a validation file and a variables file, as available. 58 | } 59 | \description{ 60 | Pull files from the NEON API, by data product, merge data for each table, and read into the current R environment 61 | } 62 | \details{ 63 | All available data meeting the query criteria will be downloaded. Most data products are collected at only a subset of sites, and dates of collection vary. Consult the NEON data portal for sampling details. 64 | Dates are specified only to the month because NEON data are provided in monthly packages. Any month included in the search criteria will be included in the download. Start and end date are inclusive. 65 | } 66 | \examples{ 67 | \dontrun{ 68 | # To download plant foliar properties data from all sites, expanded data package: 69 | cfc <- loadByProduct(dpID="DP1.10026.001", site="all", package="expanded") 70 | } 71 | } 72 | \references{ 73 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 74 | } 75 | \author{ 76 | Claire Lunch \email{clunch@battelleecology.org} 77 | } 78 | -------------------------------------------------------------------------------- /neonUtilities/man/makePosColumns.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/makePosColumns.R 3 | \name{makePosColumns} 4 | \alias{makePosColumns} 5 | \title{Create position (horizontal and vertical) columns} 6 | \usage{ 7 | makePosColumns(d, datafl, site) 8 | } 9 | \arguments{ 10 | \item{d}{A data table} 11 | 12 | \item{datafl}{A data file name} 13 | } 14 | \value{ 15 | A data table with new columns 16 | } 17 | \description{ 18 | For instrumented meteorological data products, create position (horizontal and vertical) columns based on values 19 | embedded in the file names. 20 | } 21 | \references{ 22 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 23 | } 24 | \author{ 25 | Christine Laney \email{claney@battelleecology.org} 26 | } 27 | \keyword{internal} 28 | -------------------------------------------------------------------------------- /neonUtilities/man/other_bundles.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/other_bundles.R 3 | \docType{data} 4 | \name{other_bundles} 5 | \alias{other_bundles} 6 | \title{Bundled vegetation and sediment data product information} 7 | \format{ 8 | A data frame with 2 variables: 9 | \describe{ 10 | \item{product}{Data product ID of a product} 11 | \item{homeProduct}{Data product ID of the corresponding home data product} 12 | } 13 | } 14 | \source{ 15 | NEON data product bundles 16 | } 17 | \usage{ 18 | other_bundles 19 | } 20 | \description{ 21 | A dataset containing NEON data product codes of vegetation and sediment data products and the "home" data products they are bundled with. 22 | } 23 | \keyword{datasets} 24 | -------------------------------------------------------------------------------- /neonUtilities/man/queryFiles.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/queryFiles.R 3 | \name{queryFiles} 4 | \alias{queryFiles} 5 | \title{Get a list of data files from the query endpoint of the NEON API} 6 | \usage{ 7 | queryFiles( 8 | dpID, 9 | site = "all", 10 | startdate = NA, 11 | enddate = NA, 12 | package = "basic", 13 | release = "current", 14 | timeIndex = "all", 15 | tabl = "all", 16 | metadata = TRUE, 17 | include.provisional = FALSE, 18 | token = NA_character_ 19 | ) 20 | } 21 | \arguments{ 22 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001} 23 | 24 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.} 25 | 26 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 27 | 28 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 29 | 30 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.} 31 | 32 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.} 33 | 34 | \item{timeIndex}{Either the string 'all', or the time index of data to download, in minutes. Only applicable to sensor (IS) data. Defaults to 'all'.} 35 | 36 | \item{tabl}{Either the string 'all', or the name of a single data table to download. Defaults to 'all'.} 37 | 38 | \item{metadata}{T or F, should urls for metadata files (variables, sensor positions, etc) be included. Defaults to F, can only be set to T if tabl is not 'all'.} 39 | 40 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.} 41 | 42 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.} 43 | } 44 | \value{ 45 | A list of two elements: (1) the set of urls matching the query; (2) the most recent variables file for the set of urls 46 | } 47 | \description{ 48 | Uses the query endpoint of the NEON API to find the full list of files for a given data product, release, site(s), and date range. 49 | } 50 | \references{ 51 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 52 | } 53 | \author{ 54 | Claire Lunch \email{clunch@battelleecology.org} 55 | } 56 | -------------------------------------------------------------------------------- /neonUtilities/man/quietMessages.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/quietMessages.R 3 | \name{quietMessages} 4 | \alias{quietMessages} 5 | \title{Will suppress all output messages, while retaining the output dataframe} 6 | \usage{ 7 | quietMessages(toBeQuieted) 8 | } 9 | \arguments{ 10 | \item{toBeQuieted}{Input to be quieted} 11 | } 12 | \value{ 13 | The expected output without associated messages/warnings. 14 | } 15 | \description{ 16 | Used to quiet all output messages 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Nate Mietkiewicz \email{mietkiewicz@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/readTableNEON.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/readTableNEON.R 3 | \name{readTableNEON} 4 | \alias{readTableNEON} 5 | \title{Read a NEON data table with correct data types for each variable} 6 | \usage{ 7 | readTableNEON(dataFile, varFile, useFasttime = FALSE) 8 | } 9 | \arguments{ 10 | \item{dataFile}{A data frame containing a NEON data table, or the filepath to a data table to load} 11 | 12 | \item{varFile}{A data frame containing the corresponding NEON variables file, or the filepath to the variables file} 13 | 14 | \item{useFasttime}{Should the fasttime package be used to read date-time variables? Defaults to false.} 15 | } 16 | \value{ 17 | A data frame of a NEON data table, with column classes assigned by data type 18 | } 19 | \description{ 20 | Load a table into R, assigning classes to each column based on data types in variables file; or convert a table already loaded 21 | } 22 | \references{ 23 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 24 | } 25 | \author{ 26 | Claire Lunch \email{clunch@battelleecology.org} 27 | } 28 | -------------------------------------------------------------------------------- /neonUtilities/man/schemaAllStrings.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/schemaAllStrings.R 3 | \name{schemaAllStrings} 4 | \alias{schemaAllStrings} 5 | \title{Create an arrow schema with every variable coded as a string field.} 6 | \usage{ 7 | schemaAllStrings(variables) 8 | } 9 | \arguments{ 10 | \item{variables}{A data frame containing a NEON variables file for a single table, or a set of field names.} 11 | } 12 | \value{ 13 | An arrow schema for the relevant data table with all variables set to string. 14 | } 15 | \description{ 16 | Use the field names in a NEON variables file to create an arrow schema of all strings, or, if no variables file is available, read the header and assign everything as string. 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Claire Lunch \email{clunch@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/schemaFromVar.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/schemaFromVar.R 3 | \name{schemaFromVar} 4 | \alias{schemaFromVar} 5 | \title{Create an arrow schema from a NEON variables file.} 6 | \usage{ 7 | schemaFromVar(variables, tab, package) 8 | } 9 | \arguments{ 10 | \item{variables}{A data frame containing a NEON variables file, or a url pointing to a NEON variables file.} 11 | 12 | \item{tab}{The name of the table to generate a schema from.} 13 | 14 | \item{package}{Should the schema be created for the basic or expanded package?} 15 | } 16 | \value{ 17 | An arrow schema for the relevant data table. 18 | } 19 | \description{ 20 | Use the field names and data types in a NEON variables file to create an arrow schema. 21 | } 22 | \references{ 23 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 24 | } 25 | \author{ 26 | Claire Lunch \email{clunch@battelleecology.org} 27 | } 28 | -------------------------------------------------------------------------------- /neonUtilities/man/shared_aquatic.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/shared_aquatic.R 3 | \docType{data} 4 | \name{shared_aquatic} 5 | \alias{shared_aquatic} 6 | \title{Terrestrial-aquatic shared data information} 7 | \format{ 8 | A data frame with 3 variables: 9 | \describe{ 10 | \item{site}{site code of a NEON aquatic site} 11 | \item{towerSite}{site code of the NEON terrestrial site used as the data source for the corresponding aquatic site} 12 | \item{product}{Data product ID of the data products to which the corresponding terrestrial-aquatic site relationship relates} 13 | } 14 | } 15 | \source{ 16 | NEON site layouts and spatial design 17 | } 18 | \usage{ 19 | shared_aquatic 20 | } 21 | \description{ 22 | A dataset containing NEON site codes and data product IDs for places where meteorological data from terrestrial sites are used as the data of record for nearby aquatic sites as well. 23 | } 24 | \keyword{datasets} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/shared_flights.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/shared_flights.R 3 | \docType{data} 4 | \name{shared_flights} 5 | \alias{shared_flights} 6 | \title{Flight coverage information} 7 | \format{ 8 | A data frame with 2 variables: 9 | \describe{ 10 | \item{site}{site code of a NEON site} 11 | \item{flightSite}{site code that matches the file naming for flights that may include "site"} 12 | } 13 | } 14 | \source{ 15 | NEON flight plans 16 | } 17 | \usage{ 18 | shared_flights 19 | } 20 | \description{ 21 | A dataset containing NEON site codes for places where a single AOP flight may cover multiple sites 22 | } 23 | \keyword{datasets} 24 | -------------------------------------------------------------------------------- /neonUtilities/man/stackByTable.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/stackByTable.R 3 | \name{stackByTable} 4 | \alias{stackByTable} 5 | \title{Join data files in a zipped NEON data package by table type} 6 | \usage{ 7 | stackByTable( 8 | filepath, 9 | savepath = NA, 10 | folder = FALSE, 11 | saveUnzippedFiles = FALSE, 12 | dpID = NA, 13 | package = NA, 14 | nCores = 1, 15 | useFasttime = FALSE 16 | ) 17 | } 18 | \arguments{ 19 | \item{filepath}{The location of the zip file} 20 | 21 | \item{savepath}{The location to save the output files to} 22 | 23 | \item{folder}{T or F: does the filepath point to a parent, unzipped folder, or a zip file? If F, assumes the filepath points to a zip file. Defaults to F. No longer needed; included for back compatibility.} 24 | 25 | \item{saveUnzippedFiles}{T or F: should the unzipped monthly data folders be retained?} 26 | 27 | \item{dpID}{Data product ID of product to stack. Ignored and determined from data unless input is a vector of files, generally via stackFromStore().} 28 | 29 | \item{package}{Data download package, either basic or expanded. Ignored and determined from data unless input is a vector of files, generally via stackFromStore().} 30 | 31 | \item{nCores}{The number of cores to parallelize the stacking procedure. To automatically use the maximum number of cores on your machine we suggest setting nCores=parallel::detectCores(). By default it is set to a single core.} 32 | 33 | \item{useFasttime}{Should the fasttime package be used to read date-time variables? Only relevant if savepath="envt". Defaults to false.} 34 | } 35 | \value{ 36 | All files are unzipped and one file for each table type is created and written. If savepath="envt" is specified, output is a named list of tables; otherwise, function output is null and files are saved to the location specified. 37 | } 38 | \description{ 39 | Given a zipped data file, do a full join of all data files, grouped by table type. 40 | This should result in a small number of large files. 41 | } 42 | \examples{ 43 | \dontrun{ 44 | # To unzip and merge files downloaded from the NEON Data Portal 45 | stackByTable("~/NEON_par.zip") 46 | 47 | # To unzip and merge files downloaded using zipsByProduct() 48 | stackByTable("~/filesToStack00024") 49 | } 50 | } 51 | \references{ 52 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 53 | } 54 | \author{ 55 | Christine Laney \email{claney@battelleecology.org} 56 | Claire Lunch \email{clunch@battelleecology.org} 57 | } 58 | -------------------------------------------------------------------------------- /neonUtilities/man/stackDataFilesParallel.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/stackDataFilesParallel.R 3 | \name{stackDataFilesParallel} 4 | \alias{stackDataFilesParallel} 5 | \title{Join data files in a unzipped NEON data package by table type} 6 | \usage{ 7 | stackDataFilesParallel(folder, nCores = 1, dpID) 8 | } 9 | \arguments{ 10 | \item{folder}{The location of the data} 11 | 12 | \item{nCores}{The number of cores to parallelize the stacking procedure. To automatically use the maximum number of cores on your machine we suggest setting 'nCores=parallel::detectCores()'. By default it is set to a single core. If the files are less than 25000 bytes the userdefined nCores will be overridden to a single core.} 13 | 14 | \item{dpID}{The data product identifier} 15 | } 16 | \value{ 17 | One file for each table type is created and written. 18 | } 19 | \description{ 20 | Given a folder of unzipped files (unzipped NEON data file), do a full join of all data files, grouped by table type. 21 | This should result in a small number of large files. 22 | } 23 | \references{ 24 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 25 | } 26 | \author{ 27 | Christine Laney \email{claney@battelleecology.org} 28 | } 29 | \keyword{internal} 30 | -------------------------------------------------------------------------------- /neonUtilities/man/stackEddy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/stackEddy.R 3 | \name{stackEddy} 4 | \alias{stackEddy} 5 | \title{Extract eddy covariance data from HDF5 format} 6 | \usage{ 7 | stackEddy( 8 | filepath, 9 | level = "dp04", 10 | var = NA, 11 | avg = NA, 12 | metadata = FALSE, 13 | useFasttime = FALSE, 14 | runLocal = FALSE 15 | ) 16 | } 17 | \arguments{ 18 | \item{filepath}{One of: a folder containing NEON EC H5 files, a zip file of DP4.00200.001 data downloaded from the NEON data portal, a folder of DP4.00200.001 data downloaded by the neonUtilities::zipsByProduct() function, or a single NEON EC H5 file [character]} 19 | 20 | \item{level}{The level of data to extract; one of dp01, dp02, dp03, dp04 [character]} 21 | 22 | \item{var}{The variable set to extract. Can be any of the variables in the "name" level or the "system" level of the H5 file; use the getVarsEddy() function to see the available variables. From the inputs, all variables from "name" and all variables from "system" will be returned, but if variables from both "name" and "system" are specified, the function will return only the intersecting set. This allows the user to, e.g., return only the pressure data ("pres") from the CO2 storage system ("co2Stor"), instead of all the pressure data from all instruments. [character]} 23 | 24 | \item{avg}{The averaging interval to extract, in minutes [numeric]} 25 | 26 | \item{metadata}{Should the output include metadata from the attributes of the H5 files? Defaults to false. Even when false, variable definitions, issue logs, and science review flags will be included. [logical]} 27 | 28 | \item{useFasttime}{Should the fasttime package be used to convert time stamps to time format? Decreases stacking time but can introduce imprecision at the millisecond level. Defaults to false. [logical]} 29 | 30 | \item{runLocal}{Set to TRUE to omit any calls to the NEON API. Data are extracted and reformatted from local files, but citation and issue log are not retrieved. [logical]} 31 | } 32 | \value{ 33 | A named list of data frames. One data frame per site, plus one data frame containing the metadata (objDesc) table and one data frame containing units for each variable (variables). 34 | } 35 | \description{ 36 | Convert data of choice from HDF5 to tabular format. Specific to eddy covariance data product: DP4.00200.001 37 | } 38 | \details{ 39 | Given a filepath containing H5 files of DP4.00200.001 data, extracts variables, stacks data tables over time, and joins variables into a single table. 40 | For data product levels 2-4 (dp02, dp03, dp04), joins all available data, except for the flux footprint data in the expanded package. 41 | For dp01, an averaging interval and a set of variable names must be provided as inputs. 42 | } 43 | \examples{ 44 | \dontrun{ 45 | # To extract and merge Level 4 data tables, where data files are in the working directory 46 | flux <- stackEddy(filepath=getwd(), level='dp04', var=NA, avg=NA) 47 | } 48 | } 49 | \references{ 50 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 51 | } 52 | \author{ 53 | Claire Lunch \email{clunch@battelleecology.org} 54 | } 55 | -------------------------------------------------------------------------------- /neonUtilities/man/stackFromStore.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/stackFromStore.R 3 | \name{stackFromStore} 4 | \alias{stackFromStore} 5 | \title{Select files from a stored set of NEON data, created by neonstore package methods or another method} 6 | \usage{ 7 | stackFromStore( 8 | filepaths, 9 | dpID, 10 | site = "all", 11 | startdate = NA, 12 | enddate = NA, 13 | pubdate = NA, 14 | timeIndex = "all", 15 | level = "dp04", 16 | var = NA, 17 | zipped = FALSE, 18 | package = "basic", 19 | load = TRUE, 20 | nCores = 1 21 | ) 22 | } 23 | \arguments{ 24 | \item{filepaths}{Either a vector of filepaths pointing to files to be stacked, or a single directory containing files that can be stacked, with selection criteria detmined by the other inputs. In both cases files to be stacked must be either site-month zip files or unzipped folders corresponding to site-month zips. [character]} 25 | 26 | \item{dpID}{The NEON data product ID of the data to be stacked [character]} 27 | 28 | \item{site}{Either "all" or a vector of NEON site codes to be stacked [character]} 29 | 30 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA. [character]} 31 | 32 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA. [character]} 33 | 34 | \item{pubdate}{The maximum publication date of data to include in stacking, in the form YYYY-MM-DD. If NA, the most recently published data for each product-site-month combination will be selected. Otherwise, the most recent publication date that is older than pubdate will be selected. Thus the data stacked will be the data that would have been accessed on the NEON Data Portal, if it had been downloaded on pubdate. [character]} 35 | 36 | \item{timeIndex}{Either the string 'all', or the time index of data to be stacked, in minutes. Only applicable to sensor (IS) and eddy covariance data. Defaults to 'all'. [character]} 37 | 38 | \item{level}{Data product level of data to stack. Only applicable to eddy covariance (SAE) data; see stackEddy() documentation. [character]} 39 | 40 | \item{var}{Variables to be extracted and stacked from H5 files. Only applicable to eddy covariance (SAE) data; see stackEddy() documentation. [character]} 41 | 42 | \item{zipped}{Should stacking use data from zipped files or unzipped folders? This option allows zips and their equivalent unzipped folders to be stored in the same directory; stacking will extract whichever is specified. Defaults to FALSE, i.e. stacking using unzipped folders. [logical]} 43 | 44 | \item{package}{Either "basic" or "expanded", indicating which data package to stack. Defaults to basic. [character]} 45 | 46 | \item{load}{If TRUE, stacked data are read into the current R environment. If FALSE, stacked data are written to the directory where data files are stored. Defaults to TRUE. [logical]} 47 | 48 | \item{nCores}{Number of cores to use for optional parallel processing. Defaults to 1. [integer]} 49 | } 50 | \value{ 51 | If load=TRUE, returns a named list of stacked data tables. If load=FALSE, return is empty and stacked files are written to data directory. 52 | } 53 | \description{ 54 | Select files from a stored set based on input criteria and pass to stackByTable() or stackEddy() 55 | } 56 | \references{ 57 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 58 | } 59 | \author{ 60 | Claire Lunch \email{clunch@battelleecology.org} 61 | } 62 | -------------------------------------------------------------------------------- /neonUtilities/man/table_types.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/table_types.R 3 | \docType{data} 4 | \name{table_types} 5 | \alias{table_types} 6 | \title{Publication table information} 7 | \format{ 8 | A data frame with 5 variables. Number of rows changes frequently as more tables are added: 9 | \describe{ 10 | \item{productID}{data product ID} 11 | \item{tableName}{name of table} 12 | \item{tableDesc}{description of table} 13 | \item{tableType}{type of table (important for knowing which tables to stack, and how to stack)} 14 | \item{tableTMI}{a time index (e.g., 0 = native resolution, 1 = 1 minute, 30 = 30 minute averages or totals)} 15 | } 16 | } 17 | \source{ 18 | NEON database 19 | } 20 | \usage{ 21 | table_types 22 | } 23 | \description{ 24 | A dataset containing publication table names, descriptions, type (site-date, site-all, lab-all, lab-current), and a time index 25 | } 26 | \keyword{datasets} 27 | -------------------------------------------------------------------------------- /neonUtilities/man/timeStampSet.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/timeStampSet.R 3 | \name{timeStampSet} 4 | \alias{timeStampSet} 5 | \title{Generate a consensus set of time stamps from a set of input tables.} 6 | \usage{ 7 | timeStampSet(tabList) 8 | } 9 | \arguments{ 10 | \item{tabList}{A list of SAE data tables} 11 | } 12 | \value{ 13 | A table of time stamps (start and end times) aggregated from the input tables 14 | } 15 | \description{ 16 | Generate consensus SAE time stamps from a set of tables. Used in stackEddy(), not intended for independent use. 17 | } 18 | \references{ 19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 20 | } 21 | \author{ 22 | Claire Lunch \email{clunch@battelleecology.org} 23 | } 24 | \keyword{internal} 25 | -------------------------------------------------------------------------------- /neonUtilities/man/transformFileToGeoCSV.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/transformFileToGeoCSV.R 3 | \name{transformFileToGeoCSV} 4 | \alias{transformFileToGeoCSV} 5 | \title{Transform NEON CSV file to GeoCSV} 6 | \usage{ 7 | transformFileToGeoCSV(infile, varfile, outfile) 8 | } 9 | \arguments{ 10 | \item{infile}{The path to the file that needs to be parsed} 11 | 12 | \item{varfile}{The path to the variables file needed to parse the infile} 13 | 14 | \item{outfile}{The path where the new GeoCSV file should be placed} 15 | } 16 | \value{ 17 | The same data file with a GeoCSV header 18 | } 19 | \description{ 20 | Read in a NEON monthly data zip file and parse the respective variables file to create a new GeoCSV file 21 | } 22 | \references{ 23 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 24 | } 25 | \author{ 26 | Christine Laney \email{claney@battelleecology.org} 27 | } 28 | -------------------------------------------------------------------------------- /neonUtilities/man/unzipZipfileParallel.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/unzipZipfileParallel.R 3 | \name{unzipZipfileParallel} 4 | \alias{unzipZipfileParallel} 5 | \title{Unzip a zip file either at just the top level or recursively through the file} 6 | \usage{ 7 | unzipZipfileParallel( 8 | zippath, 9 | outpath = substr(zippath, 1, nchar(zippath) - 4), 10 | level = "all", 11 | nCores = 1 12 | ) 13 | } 14 | \arguments{ 15 | \item{zippath}{The filepath of the input file} 16 | 17 | \item{outpath}{The name of the folder to save unpacked files to} 18 | 19 | \item{level}{Whether the unzipping should occur only for the 'top' zip file, or unzip 'all' recursively, or only files 'in' the folder specified} 20 | 21 | \item{nCores}{Number of cores to use for parallelization} 22 | } 23 | \description{ 24 | Unzip a zip file either at just the top level or recursively through the file 25 | } 26 | \references{ 27 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 28 | } 29 | \author{ 30 | Christine Laney \email{claney@battelleecology.org} 31 | Claire Lunch \email{clunch@battelleecology.org} 32 | } 33 | \keyword{internal} 34 | -------------------------------------------------------------------------------- /neonUtilities/man/zipsByProduct.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/zipsByProduct.R 3 | \name{zipsByProduct} 4 | \alias{zipsByProduct} 5 | \title{Get files from NEON API to feed the stackByTable() function} 6 | \usage{ 7 | zipsByProduct( 8 | dpID, 9 | site = "all", 10 | startdate = NA, 11 | enddate = NA, 12 | package = "basic", 13 | release = "current", 14 | timeIndex = "all", 15 | tabl = "all", 16 | check.size = TRUE, 17 | include.provisional = FALSE, 18 | savepath = NA, 19 | load = F, 20 | token = NA_character_, 21 | avg = NA 22 | ) 23 | } 24 | \arguments{ 25 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001} 26 | 27 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.} 28 | 29 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 30 | 31 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.} 32 | 33 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.} 34 | 35 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.} 36 | 37 | \item{timeIndex}{Either the string 'all', or the time index of data to download, in minutes. Only applicable to sensor (IS) data. Defaults to 'all'.} 38 | 39 | \item{tabl}{Either the string 'all', or the name of a single data table to download. Defaults to 'all'.} 40 | 41 | \item{check.size}{T or F, should the user approve the total file size before downloading? Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.} 42 | 43 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.} 44 | 45 | \item{savepath}{The location to save the output files to} 46 | 47 | \item{load}{T or F, are files saved locally or loaded directly? Used silently with loadByProduct(), do not set manually.} 48 | 49 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.} 50 | 51 | \item{avg}{Deprecated; use timeIndex} 52 | } 53 | \value{ 54 | A folder in the working directory (or in savepath, if specified), containing all zip files meeting query criteria. 55 | } 56 | \description{ 57 | Pull files from the NEON API, by data product, in a structure that will allow them to be stacked by the stackByTable() function 58 | } 59 | \details{ 60 | All available data meeting the query criteria will be downloaded. Most data products are collected at only a subset of sites, and dates of collection vary. Consult the NEON data portal for sampling details. 61 | Dates are specified only to the month because NEON data are provided in monthly packages. Any month included in the search criteria will be included in the download. Start and end date are inclusive. 62 | timeIndex: NEON sensor data are published at pre-determined averaging intervals, usually 1 and 30 minutes. The default download includes all available data. Download volume can be greatly reduced by downloading only the 30 minute files, if higher frequency data are not needed. Use getTimeIndex() to find the available averaging intervals for each sensor data product. 63 | } 64 | \examples{ 65 | \dontrun{ 66 | # To download plant foliar properties data from all sites, expanded data package: 67 | zipsByProduct(dpID="DP1.10026.001", site="all", package="expanded") 68 | } 69 | } 70 | \references{ 71 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 72 | } 73 | \author{ 74 | Claire Lunch \email{clunch@battelleecology.org} 75 | } 76 | -------------------------------------------------------------------------------- /neonUtilities/man/zipsByURI.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/zipsByURI.R 3 | \name{zipsByURI} 4 | \alias{zipsByURI} 5 | \title{Get files from NEON ECS Bucket using URLs in stacked data} 6 | \usage{ 7 | zipsByURI( 8 | filepath, 9 | savepath = paste0(filepath, "/ECS_zipFiles"), 10 | pick.files = FALSE, 11 | check.size = TRUE, 12 | unzip = TRUE, 13 | saveZippedFiles = FALSE, 14 | token = NA_character_ 15 | ) 16 | } 17 | \arguments{ 18 | \item{filepath}{The location of the NEON data containing URIs. Can be either a local directory containing NEON tabular data or a list object containing tabular data.} 19 | 20 | \item{savepath}{The location to save the output files from the ECS bucket, optional. 21 | Defaults to creating a "ECS_zipFiles" folder in the filepath directory.} 22 | 23 | \item{pick.files}{T or F, should the user be told the name of each file before downloading? 24 | Defaults to F. When working in batch mode, or other non-interactive workflow, use pick.files=F.} 25 | 26 | \item{check.size}{T or F, should the user be told the total file size before downloading? 27 | Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.} 28 | 29 | \item{unzip}{T or F, indicates if the downloaded zip files from ECS buckets should be 30 | unzipped into the same directory, defaults to T. Supports .zip and .tar.gz files currently.} 31 | 32 | \item{saveZippedFiles}{T or F: should the zip files be retained after unzipping? Defaults to F.} 33 | 34 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.} 35 | } 36 | \value{ 37 | A folder in the working directory (or in savepath, if specified), containing all files meeting query criteria. 38 | } 39 | \description{ 40 | Read in a set of URLs from NEON data tables and then download the data from the NEON ECS buckets. 41 | Assumes data tables are in the format resulting from merging files using stackByTable(). 42 | File downloads from ECS can be extremely large; be prepared for long download times and large file storage. 43 | } 44 | \examples{ 45 | \dontrun{ 46 | # To download stream morphology data from stacked data: 47 | zipsByURI(filepath="~/filesToStack00131/stackedFiles") 48 | } 49 | } 50 | \references{ 51 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 52 | } 53 | \author{ 54 | Kaelin Cawley \email{kcawley@battelleecology.org} 55 | } 56 | -------------------------------------------------------------------------------- /neonUtilities/neonUtilities.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | 18 | BuildType: Package 19 | PackageUseDevtools: Yes 20 | PackageInstallArgs: --no-multiarch --with-keep.source 21 | -------------------------------------------------------------------------------- /neonUtilities/tests/testthat.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | library(neonUtilities) 3 | 4 | test_check("neonUtilities") 5 | -------------------------------------------------------------------------------- /neonUtilities/tests/testthat/test_byTileAOP.R: -------------------------------------------------------------------------------- 1 | context("byTileAOP") 2 | library(neonUtilities) 3 | 4 | test_that("Test that unequal length easting and northing vectors return error", { 5 | expect_error(byTileAOP(dpID="DP3.30015.001", site="WREF", year=2017, 6 | easting=571000, northing=c(5079000,5080000)), 7 | "Easting and northing vector lengths do not match, and/or contain null values. Cannot identify paired coordinates.") 8 | }) 9 | -------------------------------------------------------------------------------- /neonUtilities/tests/testthat/test_stackByTable.R: -------------------------------------------------------------------------------- 1 | context("Table stacking") 2 | library(neonUtilities) 3 | 4 | files_match <- function(object, output, expected) { 5 | object 6 | comp <- compare(output$uid, 7 | expected$uid) 8 | expect( 9 | comp$equal, 10 | sprintf("Output files do not match previous version files.\n%s", comp$message) 11 | ) 12 | } 13 | 14 | f <- stackByTable(system.file("extdata", "NEON_gp.zip", 15 | package="neonUtilities"), savepath="envt") 16 | test_that("Test that correct files are created for microbe group abundances", { 17 | files_match({}, 18 | f$mga_soilGroupAbundances, 19 | read.csv(system.file("extdata", "expected/gp/mga_soilGroupAbundances.csv", 20 | package="neonUtilities"))) 21 | }) 22 | 23 | -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.0.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.0.0.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.0.1.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.2.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.2.0.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.2.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.2.1.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.2.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.2.2.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.0.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.1.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.2.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.3.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.4.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.5.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.5.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.6.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.6.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.7.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.7.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.8.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.8.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_1.3.9.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.9.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.0.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.0.0.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.0.1.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.0.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.1.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.1.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.1.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.2.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.1.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.3.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.1.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.4.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.2.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.2.0.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.2.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.2.1.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.3.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.3.0.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.4.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.4.0.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.4.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.4.1.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.4.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.4.2.tar.gz -------------------------------------------------------------------------------- /release_archive/neonUtilities_2.4.3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.4.3.tar.gz --------------------------------------------------------------------------------