├── .github
└── ISSUE_TEMPLATE
│ ├── bug_report.md
│ └── feature_request.md
├── .gitignore
├── LICENSE.txt
├── README.md
├── cheat-sheet-neonUtilities.key
├── cheat-sheet-neonUtilities.pdf
├── helper_files
├── added_fields.csv
├── chem_bundles.csv
├── other_bundles.csv
├── readme.md
├── release_2021.csv
├── shared_aquatic.csv
├── shared_flights.csv
└── table_types.csv
├── logo.png
├── neonUtilities
├── .Rbuildignore
├── DESCRIPTION
├── LICENSE
├── NAMESPACE
├── NEWS.md
├── R
│ ├── assignClasses.R
│ ├── byEventSIM.R
│ ├── byFileAOP.R
│ ├── byTileAOP.R
│ ├── chem_bundles.R
│ ├── cleanUp.R
│ ├── convByteSize.R
│ ├── datasetQuery.R
│ ├── dateConvert.R
│ ├── eddyStampCheck.R
│ ├── findDatatables.R
│ ├── findTablesByFormat.R
│ ├── findTablesUnique.R
│ ├── footRaster.R
│ ├── getAPI.R
│ ├── getAPIHeaders.R
│ ├── getAttributes.R
│ ├── getAvg.R
│ ├── getCitation.R
│ ├── getDatatable.R
│ ├── getEddyLog.R
│ ├── getFileUrls.R
│ ├── getIssueLog.R
│ ├── getNeonDOI.R
│ ├── getPackage.R
│ ├── getProductInfo.R
│ ├── getProductSensors.R
│ ├── getReadmePublicationDate.R
│ ├── getRecentPublication.R
│ ├── getTaxonTable.R
│ ├── getTileUrls.R
│ ├── getTimeIndex.R
│ ├── getTitle.R
│ ├── getVariables.R
│ ├── getVariablesEddy.R
│ ├── getVarsEddy.R
│ ├── getZipUrls.R
│ ├── listFilesInZip.R
│ ├── listZipfiles.R
│ ├── loadByProduct.R
│ ├── makePosColumns.R
│ ├── other_bundles.R
│ ├── queryFiles.R
│ ├── quietMessages.R
│ ├── readTableNEON.R
│ ├── schemaAllStrings.R
│ ├── schemaFromVar.R
│ ├── shared_aquatic.R
│ ├── shared_flights.R
│ ├── stackByTable.R
│ ├── stackDataFilesParallel.R
│ ├── stackEddy.R
│ ├── stackFromStore.R
│ ├── sysdata.rda
│ ├── table_types.R
│ ├── timeStampSet.R
│ ├── transformFileToGeoCSV.R
│ ├── unzipZipfileParallel.R
│ ├── zipsByProduct.R
│ └── zipsByURI.R
├── README.md
├── data-raw
│ └── update_table_types.R
├── data
│ ├── chem_bundles.rda
│ ├── other_bundles.rda
│ ├── shared_aquatic.rda
│ ├── shared_flights.rda
│ └── table_types.rda
├── inst
│ └── extdata
│ │ ├── NEON_gp.zip
│ │ └── expected
│ │ └── gp
│ │ ├── issueLog_10109.csv
│ │ ├── mga_batchResults.csv
│ │ ├── mga_labSummary.csv
│ │ ├── mga_soilGroupAbundances.csv
│ │ ├── readme_10109.txt
│ │ ├── validation_10109.csv
│ │ └── variables_10109.csv
├── man
│ ├── assignClasses.Rd
│ ├── byEventSIM.Rd
│ ├── byFileAOP.Rd
│ ├── byTileAOP.Rd
│ ├── chem_bundles.Rd
│ ├── cleanUp.Rd
│ ├── convByteSize.Rd
│ ├── datasetQuery.Rd
│ ├── dateConvert.Rd
│ ├── eddyStampCheck.Rd
│ ├── findDatatables.Rd
│ ├── findTablesByFormat.Rd
│ ├── findTablesUnique.Rd
│ ├── footRaster.Rd
│ ├── getAPI.Rd
│ ├── getAPIHeaders.Rd
│ ├── getAttributes.Rd
│ ├── getAvg.Rd
│ ├── getCitation.Rd
│ ├── getDatatable.Rd
│ ├── getEddyLog.Rd
│ ├── getFileUrls.Rd
│ ├── getIssueLog.Rd
│ ├── getNeonDOI.Rd
│ ├── getPackage.Rd
│ ├── getProductInfo.Rd
│ ├── getProductSensors.Rd
│ ├── getReadmePublicationDate.Rd
│ ├── getRecentPublication.Rd
│ ├── getTaxonTable.Rd
│ ├── getTileUrls.Rd
│ ├── getTimeIndex.Rd
│ ├── getTitle.Rd
│ ├── getVariables.Rd
│ ├── getVariablesEddy.Rd
│ ├── getVarsEddy.Rd
│ ├── getZipUrls.Rd
│ ├── listFilesInZip.Rd
│ ├── listZipfiles.Rd
│ ├── loadByProduct.Rd
│ ├── makePosColumns.Rd
│ ├── other_bundles.Rd
│ ├── queryFiles.Rd
│ ├── quietMessages.Rd
│ ├── readTableNEON.Rd
│ ├── schemaAllStrings.Rd
│ ├── schemaFromVar.Rd
│ ├── shared_aquatic.Rd
│ ├── shared_flights.Rd
│ ├── stackByTable.Rd
│ ├── stackDataFilesParallel.Rd
│ ├── stackEddy.Rd
│ ├── stackFromStore.Rd
│ ├── table_types.Rd
│ ├── timeStampSet.Rd
│ ├── transformFileToGeoCSV.Rd
│ ├── unzipZipfileParallel.Rd
│ ├── zipsByProduct.Rd
│ └── zipsByURI.Rd
├── neonUtilities.Rproj
└── tests
│ ├── testthat.R
│ └── testthat
│ ├── test_byTileAOP.R
│ └── test_stackByTable.R
└── release_archive
├── neonUtilities_1.0.0.tar.gz
├── neonUtilities_1.0.1.tar.gz
├── neonUtilities_1.2.0.tar.gz
├── neonUtilities_1.2.1.tar.gz
├── neonUtilities_1.2.2.tar.gz
├── neonUtilities_1.3.0.tar.gz
├── neonUtilities_1.3.1.tar.gz
├── neonUtilities_1.3.2.tar.gz
├── neonUtilities_1.3.3.tar.gz
├── neonUtilities_1.3.4.tar.gz
├── neonUtilities_1.3.5.tar.gz
├── neonUtilities_1.3.6.tar.gz
├── neonUtilities_1.3.7.tar.gz
├── neonUtilities_1.3.8.tar.gz
├── neonUtilities_1.3.9.tar.gz
├── neonUtilities_2.0.0.tar.gz
├── neonUtilities_2.0.1.tar.gz
├── neonUtilities_2.1.0.tar.gz
├── neonUtilities_2.1.1.tar.gz
├── neonUtilities_2.1.2.tar.gz
├── neonUtilities_2.1.3.tar.gz
├── neonUtilities_2.1.4.tar.gz
├── neonUtilities_2.2.0.tar.gz
├── neonUtilities_2.2.1.tar.gz
├── neonUtilities_2.3.0.tar.gz
├── neonUtilities_2.4.0.tar.gz
├── neonUtilities_2.4.1.tar.gz
├── neonUtilities_2.4.2.tar.gz
└── neonUtilities_2.4.3.tar.gz
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Bug report
3 | about: Alert us to a problem
4 | title: ''
5 | labels: bug
6 | assignees: ''
7 |
8 | ---
9 |
10 | **Function**
11 | Which of the functions in the neonUtilities package is this bug found in?
12 |
13 | **Describe the bug**
14 | A clear and concise description of what the bug is.
15 |
16 | **To Reproduce**
17 | What function inputs revealed the bug?
18 |
19 | **Expected behavior**
20 | A clear and concise description of what you expected to happen.
21 |
22 | **System (please complete the following information):**
23 | - OS: [e.g. iOS]
24 | - OS Version: [e.g. 22]
25 | - R Version: [e.g. 3.5.1]
26 |
27 | **Additional context**
28 | Add any other context about the problem here.
29 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature_request.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Feature request
3 | about: Suggest new functionality
4 | title: ''
5 | labels: enhancement
6 | assignees: ''
7 |
8 | ---
9 |
10 | **Is your feature request related to a problem? Please describe.**
11 | A clear and concise description of what the problem is.
12 |
13 | **Describe the solution you'd like**
14 | A clear and concise description of what you want to happen.
15 |
16 | **Describe alternatives you've considered**
17 | A clear and concise description of any alternative solutions or features you've considered.
18 |
19 | **Additional context**
20 | Add any other context or screenshots about the feature request here.
21 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .Rproj.user
2 | .Rhistory
3 | .RData
4 | .Ruserdata
5 | .DS_Store
6 | *.Rproj
7 | cheat-sheet-template.pptx
8 | neonUtilities-cheat-sheet.pptx
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | NEON-utilities
2 | ================
3 |
4 |
5 |
6 |
7 | NEON-utilities is a repository hosting packages and code for basic data handling for NEON data. It currently contains:
8 |
9 | 1. `neonUtilities`: an R package for discovering, downloading, and working with NEON data files. This builds upon the deprecated pacakge, `neonDataStackR`.
10 | 2. `neon-data-stacker-python`: a Python script for combining the monthly files delivered by the NEON data portal into a single file for each table type. This script is a prototype version and not for general use. It is being **deprecated** in favor of using the `neonUtilities` R package within Python. Please see the Notebook in the neon-data-stacker-python directory or use the NEON Science Using NEON Utilities in Python tutorial to learn more.
11 |
12 |
13 | Requests
14 | --------
15 |
16 | Bug reports and feature requests should be posted to the Issues page in this repository. To see current priorities for package improvement, see the neonUtilities project on the NEON public code projects board.
17 |
18 |
19 |
20 | Usage
21 | -----
22 |
23 | Specific instructions for using each code/package are provided in the README in each code/package subdirectory.
24 |
25 | If you encounter problems with code in this repository, feel free to post an issue.
26 |
27 |
28 | Credits & Acknowledgements
29 | --------------------------
30 |
31 |
32 | The National Ecological Observatory Network is a project solely funded by the National Science Foundation and managed under cooperative agreement by Battelle. Any opinions, findings, and conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the National Science Foundation.
33 |
34 |
35 | License
36 | -------
37 | GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
38 |
39 |
40 | Disclaimer
41 | ----------
42 | *Information and documents contained within this repository are available as-is. Codes or documents, or their use, may not be supported or maintained under any program or service and may not be compatible with data currently available from the NEON Data Portal.*
43 |
44 |
--------------------------------------------------------------------------------
/cheat-sheet-neonUtilities.key:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/cheat-sheet-neonUtilities.key
--------------------------------------------------------------------------------
/cheat-sheet-neonUtilities.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/cheat-sheet-neonUtilities.pdf
--------------------------------------------------------------------------------
/helper_files/added_fields.csv:
--------------------------------------------------------------------------------
1 | fieldName,description,dataType,units,downloadPkg,pubFormat,primaryKey,categoricalCodeName
2 | domainID,Unique identifier of the NEON domain,string,,appended by stackByTable,,N,""
3 | siteID,NEON site code,string,,appended by stackByTable,,N,""
4 | horizontalPosition,Index of horizontal location at a NEON site,string,,appended by stackByTable,,N,""
5 | verticalPosition,Index of vertical location at a NEON site,string,,appended by stackByTable,,N,""
6 | publicationDate,Date of data publication on the NEON data portal,dateTime,,appended by stackByTable,,N,""
7 | release,Identifier for data release,string,,appended by stackByTable,,N,""
8 |
--------------------------------------------------------------------------------
/helper_files/chem_bundles.csv:
--------------------------------------------------------------------------------
1 | product,homeProduct
2 | DP1.10102.001,depends
3 | DP1.10099.001,depends
4 | DP1.10053.001,DP1.10026.001
5 | DP1.10031.001,DP1.10033.001
6 | DP1.10101.001,DP1.10033.001
7 | DP1.10080.001,DP1.10086.001
8 | DP1.10078.001,DP1.10086.001
9 | DP1.10100.001,DP1.10086.001
10 | DP1.10008.001,DP1.10047.001
11 | DP1.00097.001,DP1.00096.001
12 |
--------------------------------------------------------------------------------
/helper_files/other_bundles.csv:
--------------------------------------------------------------------------------
1 | product,homeProduct
2 | DP1.10045.001,DP1.10098.001
3 | DP1.20197.001,DP1.20194.001
4 |
--------------------------------------------------------------------------------
/helper_files/readme.md:
--------------------------------------------------------------------------------
1 | ## Helper files for neonUtilities functionality. Additional documentation in R package manual.
2 |
3 | * added\_fields
4 |
5 | This table contains variables file metadata for the additional fields that can be added to tabular data during stacking by stackByTable(). The relevant fields are appended to the variables file as needed during the stacking process.
6 |
7 | * chem\_bundles
8 | * other\_bundles
9 | * release\_2021
10 |
11 | Several data products are "bundled" in that, e.g., litter chemical properties can only be downloaded as part of the litterfall data product. The chem\_bundles and other\_bundles tables contain the mapping, so that the download function can redirect to the bundled product if a non-independent product is requested.
12 |
13 | Some of these bundles were created after the 2021 release. The non-independent data tables are no longer accessible via the database query that populates the table\_types table, so the release\_2021 table retains them and is appended to table\_types every time table\_types is regenerated. This enables neonUtilities to continue stacking data from RELEASE-2021 correctly.
14 |
15 | * shared\_aquatic
16 | * shared\_flights
17 |
18 | Some sites are paired for some purposes. For aquatic sites that don't have a meteorological station, shared_aquatic provides the mapping to the nearby terrestrial site to be used for specific data products. The download functions redirect to the nearby terrestrial site if the aquatic site and one of the specified products are requested.
19 |
20 | Some flight boxes cover more than one site. If AOP data are requested for a site that is covered by another site's flight box, shared\_flights provides the mapping and redirects the download to the designated site.
21 |
22 | * table\_types
23 |
24 | There are a few different options for data publication from the NEON pipeline, and they have different implications for data stacking. Data that are published based on the date and site of record in the data (site-date) are straightforward, all available data are stacked. For site-all data tables, all available data are published in every data package for a given site, regardless of date. This is done when the data are not specific to a date, for example the trap establishment data for litter traps. In this case data stacking is performed on the most recently published data for each site. Data that are specific to a lab, such as method detection limits, are published in every data package that contains relevant data from the lab in question. In this case data stacking is performed on the most recently published data for each lab. The table\_types table contains an index of all NEON tabular data tables and their types. If a table is not found on this list, the findTablesByFormat() function attempts to infer the type based on the format of the file name, but the table_types table is the definitive source.
25 |
--------------------------------------------------------------------------------
/helper_files/release_2021.csv:
--------------------------------------------------------------------------------
1 | productID,tableName,tableDesc,tableType,tableTMI
2 | DP1.10045.001,vst_perplotperyear,"Per plot sampling metadata, including presence/absence of each growthForm",site-date,0
3 | DP1.10045.001,nst_perindividual,"Field measurements of individual non-herbaceous perennial plants (e.g. cacti, ferns)",site-date,0
4 | DP1.20197.001,asc_fieldDataPoint,Point level sediment collection field data,site-date,0
5 | DP1.20197.001,asc_fieldDataZone,Zone level sediment collection field data,site-date,0
6 | DP1.20197.001,asc_fieldDataStation,Station level sediment collection field data,site-date,0
7 | DP1.20197.001,asp_externalLabData,Sediment physical properties external lab data,site-date,0
8 | DP1.20197.001,asc_externalLabSummary,Sediment chemistry external lab long-term summary data,lab-current,0
9 |
--------------------------------------------------------------------------------
/helper_files/shared_aquatic.csv:
--------------------------------------------------------------------------------
1 | site,towerSite,product
2 | HOPB,HARV,DP1.00006.001
3 | HOPB,HARV,DP1.00038.001
4 | HOPB,HARV,DP1.00013.001
5 | POSE,SCBI,DP1.00006.001
6 | POSE,SCBI,DP1.00038.001
7 | POSE,SCBI,DP1.00013.001
8 | LEWI,BLAN,DP1.00006.001
9 | LEWI,BLAN,DP1.00038.001
10 | LEWI,BLAN,DP1.00013.001
11 | BARC,OSBS,DP1.00006.001
12 | BARC,OSBS,DP1.00038.001
13 | BARC,OSBS,DP1.00013.001
14 | SUGG,OSBS,DP1.00006.001
15 | SUGG,OSBS,DP1.00038.001
16 | SUGG,OSBS,DP1.00013.001
17 | FLNT,JERC,DP1.00006.001
18 | FLNT,JERC,DP1.00038.001
19 | FLNT,JERC,DP1.00013.001
20 | CRAM,UNDE,DP1.00006.001
21 | CRAM,UNDE,DP1.00038.001
22 | CRAM,UNDE,DP1.00013.001
23 | LIRO,UNDE,DP1.00006.001
24 | LIRO,UNDE,DP1.00038.001
25 | LIRO,UNDE,DP1.00013.001
26 | KING,KONZ,DP1.00006.001
27 | KING,KONZ,DP1.00038.001
28 | KING,KONZ,DP1.00013.001
29 | WALK,ORNL,DP1.00006.001
30 | WALK,ORNL,DP1.00038.001
31 | WALK,ORNL,DP1.00013.001
32 | LECO,GRSM,DP1.00006.001
33 | LECO,GRSM,DP1.00038.001
34 | LECO,GRSM,DP1.00013.001
35 | MAYF,TALL,DP1.00006.001
36 | MAYF,TALL,DP1.00038.001
37 | MAYF,TALL,DP1.00013.001
38 | BLWA,DELA,DP1.00004.001
39 | BLWA,DELA,DP1.00001.001
40 | BLWA,DELA,DP1.00024.001
41 | BLWA,DELA,DP1.00098.001
42 | BLWA,DELA,DP1.00002.001
43 | BLWA,DELA,DP1.00023.001
44 | BLWA,DELA,DP1.00006.001
45 | BLWA,DELA,DP1.00038.001
46 | BLWA,DELA,DP1.00013.001
47 | TOMB,LENO,DP1.00004.001
48 | TOMB,LENO,DP1.00001.001
49 | TOMB,LENO,DP1.00024.001
50 | TOMB,LENO,DP1.00098.001
51 | TOMB,LENO,DP1.00002.001
52 | TOMB,LENO,DP1.00023.001
53 | TOMB,LENO,DP1.00006.001
54 | TOMB,LENO,DP1.00038.001
55 | TOMB,LENO,DP1.00013.001
56 | PRPO,WOOD,DP1.00006.001
57 | PRPO,WOOD,DP1.00038.001
58 | PRPO,WOOD,DP1.00013.001
59 | PRLA,DCFS,DP1.00006.001
60 | PRLA,DCFS,DP1.00038.001
61 | PRLA,DCFS,DP1.00013.001
62 | BLDE,YELL,DP1.00006.001
63 | BLDE,YELL,DP1.00038.001
64 | BLDE,YELL,DP1.00013.001
65 | COMO,NIWO,DP1.00006.001
66 | COMO,NIWO,DP1.00038.001
67 | COMO,NIWO,DP1.00013.001
68 | MART,WREF,DP1.00006.001
69 | MART,WREF,DP1.00038.001
70 | MART,WREF,DP1.00013.001
71 | TECR,TEAK,DP1.00006.001
72 | TECR,TEAK,DP1.00038.001
73 | TECR,TEAK,DP1.00013.001
74 | OKSR,TOOL,DP1.00006.001
75 | OKSR,TOOL,DP1.00038.001
76 | OKSR,TOOL,DP1.00013.001
77 | CARI,BONA,DP1.00006.001
78 | CARI,BONA,DP1.00038.001
79 | CARI,BONA,DP1.00013.001
80 |
--------------------------------------------------------------------------------
/helper_files/shared_flights.csv:
--------------------------------------------------------------------------------
1 | site,flightSite
2 | TREE,STEI
3 | CHEQ,STEI
4 | KONA,KONZ
5 | BARC,OSBS
6 | SUGG,OSBS
7 | COMO,NIWO
8 | MAYF,TALL
9 | FLNT,JERC
10 | TOMB,LENO
11 | BLWA,DELA
12 | POSE,SCBI
13 | CRAM,UNDE
14 | TOOK,TOOL
15 | CARI,BONA
16 | MART,WREF
17 | TECR,TEAK
18 | BIGC,SOAP
19 | BLDE,YELL
20 | PRLA,WOOD
21 | PRPO,WOOD
22 | KING,KONZ
23 | LEWI,BLAN
24 | DCFS,WOOD
25 |
--------------------------------------------------------------------------------
/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/logo.png
--------------------------------------------------------------------------------
/neonUtilities/.Rbuildignore:
--------------------------------------------------------------------------------
1 | ^.*\.Rproj$
2 | ^\.Rproj\.user$
3 | ^data-raw$
4 | ^LICENSE$
5 |
--------------------------------------------------------------------------------
/neonUtilities/DESCRIPTION:
--------------------------------------------------------------------------------
1 | Package: neonUtilities
2 | Version: 3.0.0.9000
3 | Date: 2025-02-04
4 | Title: Utilities for Working with NEON Data
5 | Authors@R: c(person("Claire","Lunch",email="clunch@battelleecology.org",role=c("aut","cre","ctb")), person("Christine","Laney",role=c("aut","ctb")), person("Nathan","Mietkiewicz",role=c("aut","ctb")), person("Eric","Sokol",role=c("aut","ctb")), person("Kaelin","Cawley",role=c("aut","ctb")), person("NEON (National Ecological Observatory Network)", role="aut"))
6 | Description: NEON data packages can be accessed through the NEON Data Portal
7 | or through the NEON Data API (see for documentation). Data delivered from
8 | the Data Portal are provided as monthly zip files packaged within a parent zip file, while individual files
9 | can be accessed from the API. This package provides tools that aid in discovering, downloading, and reformatting
10 | data prior to use in analyses. This includes downloading data via the API, merging data tables by type, and
11 | converting formats. For more information, see the readme file at .
12 | Depends: R (>= 3.6)
13 | Imports: httr, jsonlite, downloader, data.table, utils, R.utils, stats, tidyr, stringr, pbapply, parallel, curl, arrow, rlang
14 | Suggests: rhdf5, terra, testthat, fasttime, dplyr
15 | License: AGPL-3
16 | URL: https://github.com/NEONScience/NEON-utilities
17 | BugReports: https://github.com/NEONScience/NEON-utilities/issues
18 | Encoding: UTF-8
19 | LazyData: true
20 | RoxygenNote: 7.3.2
21 |
--------------------------------------------------------------------------------
/neonUtilities/NAMESPACE:
--------------------------------------------------------------------------------
1 | # Generated by roxygen2: do not edit by hand
2 |
3 | export(byEventSIM)
4 | export(byFileAOP)
5 | export(byTileAOP)
6 | export(convByteSize)
7 | export(datasetQuery)
8 | export(footRaster)
9 | export(getAvg)
10 | export(getCitation)
11 | export(getDatatable)
12 | export(getIssueLog)
13 | export(getNeonDOI)
14 | export(getPackage)
15 | export(getProductInfo)
16 | export(getProductSensors)
17 | export(getTaxonTable)
18 | export(getTimeIndex)
19 | export(getVarsEddy)
20 | export(loadByProduct)
21 | export(queryFiles)
22 | export(readTableNEON)
23 | export(schemaFromVar)
24 | export(stackByTable)
25 | export(stackEddy)
26 | export(stackFromStore)
27 | export(transformFileToGeoCSV)
28 | export(zipsByProduct)
29 | export(zipsByURI)
30 | import(httr)
31 | import(jsonlite)
32 | importFrom(httr,GET)
33 | importFrom(jsonlite,fromJSON)
34 | importFrom(rlang,.data)
35 |
--------------------------------------------------------------------------------
/neonUtilities/NEWS.md:
--------------------------------------------------------------------------------
1 | # version 2.4.3
2 |
3 | Released 2024-12-04
4 |
5 | ## Changes
6 |
7 | * stackByTable() and loadByProduct() updated to handle new files associated with Bathymetric and morphological maps (DP4.00132.001) and the three revised microbe community data products (DP1.10081.002, DP1.20086.002, and DP1.20141.002)
8 |
9 |
10 | # version 2.4.2
11 |
12 | Released 2024-04-30
13 |
14 | ## Bug fixes
15 |
16 | * stackEddy() had been erroring out any time it encountered a file that was incompatible with the input criteria (e.g., a variable requested in the var= parameter was not present in an h5 file). Now it skips the file and moves on to the next.
17 | * Download functions (zipsByProduct(), loadByProduct(), byFileAOP(), byTileAOP()) include a re-attempt routine if the first download attempt fails. include.provisional= was not handled correctly in the re-attempt code and resulted in the download erroring out. This is now fixed.
18 | * Downloads by averaging interval (timeIndex=) in zipsByProduct() and loadByProduct() were not retrieving science review flag files. This is now fixed.
19 |
20 |
21 | # version 2.4.1
22 |
23 | Released 2024-01-09
24 |
25 | ## Enhancements
26 |
27 | * New option to run stackEddy() only on already-downloaded data, without pinging the NEON API. Using this option blocks retrieval of the issue log and citation files; its use is not recommended unless you need to run code offline or otherwise need to avoid API calls. Toggle this option using the input `runLocal=`, which defaults to FALSE.
28 |
29 | ## Bug fixes
30 |
31 | * Data tables in Breeding landbird point counts (DP1.10003.001), Bathymetric and morphological maps (DP4.00132.001), and Stream morphology map (DP4.00131.001) have changed publication paradigm over time (e.g., from publication only in the site and month when collection occurred, to publication in all months with data for the relevant site). stackByTable() and loadByProduct() have been updated to accommodate both current data and past releases.
32 | * Surface-atmosphere exchange (Eddy covariance bundle, DP4.00200.001) metadata have updated the formatting of UTM zone reporting; footRaster() has been updated to handle both current and past formatting.
33 |
34 |
35 | # version 2.4.0
36 |
37 | Released 2023-10-17
38 |
39 | ## Changes
40 |
41 | * Data download functions now default to not returning provisional data. This applies to zipsByProduct(), loadByProduct(), byFileAOP(), and byTileAOP(). To download provisional data, add the new parameter `include.provisional=TRUE` to the function call.
42 | * Data stacking has been updated to handle the new formatting of the Reaeration (DP1.20190.001) and Salt-based discharge (DP1.20193.001) data products. The conductivity data tables in these two products have been converted to separate files per sampling event. This update enables stackByTable() and loadByProduct() to work with both the previous and current formats.
43 |
44 |
45 | # version 2.3.0
46 |
47 | Released 2023-07-05
48 |
49 | ## Enhancements
50 |
51 | * Data download and stacking functions now include recommended citation(s) in BibTeX format. Applies to stackByTable(), loadByProduct(), stackEddy(), byFileAOP(), and byTileAOP(). Note that citation retrieval will generally work on data downloaded from the NEON data portal or using neonUtilities functions, but if you use alternative download methods and/or modify the contents of downloaded data packages, the stacking functions may not be able to identify data release information and may not provide citations.
52 | * New parameter metadata= in stackEddy() to retrieve site, code, and CO2 validation data from H5 file attributes. When used with basic package data, the attributes are valid for the first day of each month accessed. To get attributes for each day of data, use with the expanded package.
53 | * Performance improvements to stackEddy(), will mostly be noticeable when stacking large amounts of data.
54 | * Conversion of dates to POSIXct in stacking functions can optionally use the fasttime package, toggle on and off with useFasttime=. Applies to stackByTable(), readTableNEON(), loadByProduct(), and stackEddy(). Default is not to use fasttime, since it can introduce small inconsistencies in time stamps at the millisecond level. But note that for most NEON data products, precision at the millisecond level is unlikely to be scientifically meaningful.
55 |
56 | ## Bug fixes
57 |
58 | * When no data are available for a particular day and sensor in the eddy covariance system, a single empty for the day is reported in the H5 file. In some cases, stackEddy() was mishandling these records and creating duplicate records. This is now resolved.
59 |
60 | ## Other changes
61 |
62 | * byTileAOP() and footRaster() now use the terra package for spatial data handling, replacing the raster package, which will be obsolete as of Oct 2023.
63 |
64 |
65 | -----------------------
66 |
67 | # 2023-06-30: created NEWS file
68 |
--------------------------------------------------------------------------------
/neonUtilities/R/assignClasses.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Assign correct column classes
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Use the variables file to assign classes to each column in each data file
9 | #'
10 | #' @keywords internal
11 | #' @param dt A data frame
12 | #' @param inVars The variables expected in the df
13 | #' @return A data frame with corrected column classes
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # Changelog and author contributions / copyrights
19 | # 2017-09-28 (Christine Laney): Created original function
20 | # 2018-04-13 (Christine Laney): stackDataFiles now generates a data.table rather than a data.frame;
21 | # convert back to data.frame for checking and updating column types, then convert back to data.table.
22 | ##############################################################################################
23 |
24 | assignClasses <- function(dt, inVars){
25 | df <- as.data.frame(dt)
26 | for(i in 1:ncol(df)){
27 | if(names(df)[i] %in% inVars$fieldName){
28 | type <- inVars$colClass[which(inVars$fieldName==names(df)[i])[1]]
29 | if(type == "numeric")
30 | df[,i] <- as.numeric(df[,i])
31 | if(type == "character")
32 | df[,i] <- as.character(df[,i])
33 | }
34 | }
35 | dt <- data.table::as.data.table(df)
36 | return(dt)
37 | }
38 |
--------------------------------------------------------------------------------
/neonUtilities/R/byEventSIM.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get site management data by event type.
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Query site management data to return records matching a specific eventType.
9 | #'
10 | #' @param eventType The value of eventType to search for. Can be multiple values. See categoricalCodes file for DP1.10111.001 for possible values.
11 | #' @param site Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.
12 | #' @param startdate Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.
13 | #' @param enddate Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.
14 | #' @param release The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.
15 | #' @param include.provisional T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.
16 | #' @param token User specific API token (generated within data.neonscience.org user accounts)
17 |
18 | #' @return A data frame of sim_eventData data, matching the query criteria. Note that metadata are not included in the data returned via this function.
19 |
20 | #' @examples
21 | #' \dontrun{
22 | #' # Search for fires across all NEON event data
23 | #' sim.fires <- byEventSIM(eventType="fire")
24 | #'
25 | #' # Search for grazing events at several sites
26 | #' sim.graz <- byEventSIM(eventType="grazing", site=c("CPER","KONA","MOAB","STER","LAJA"))
27 | #' }
28 | #'
29 | #' @importFrom rlang .data
30 |
31 | #' @references
32 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
33 |
34 | #' @export
35 |
36 | # changelog and author contributions / copyrights
37 | # Claire Lunch (2025-03-11)
38 | # original creation
39 | ##############################################################################################
40 |
41 | byEventSIM <- function(eventType,
42 | site="all",
43 | startdate=NA,
44 | enddate=NA,
45 | release="current",
46 | include.provisional=FALSE,
47 | token=NA_character_) {
48 |
49 | # check for dplyr
50 | if(!requireNamespace("dplyr", quietly=T)) {
51 | stop("The dplyr package is required to use this function. Install and re-try.")
52 | }
53 |
54 | urlset <- queryFiles(dpID="DP1.10111.001", site=site,
55 | package='basic', startdate=startdate,
56 | enddate=enddate, release=release,
57 | tabl='sim_eventData', metadata=TRUE,
58 | include.provisional=include.provisional,
59 | token=token)
60 |
61 | # subset to only the table to query
62 | edlst <- base::grep("sim_eventData", urlset[[1]], value=TRUE)
63 |
64 | # get variables file and translate to schema
65 | varfl <- urlset[[2]]
66 | vschema <- schemaFromVar(varfl, tab="sim_eventData", package="basic")
67 |
68 | # open dataset and query for event type
69 | ds <- arrow::open_csv_dataset(sources=edlst, schema=vschema, skip=1)
70 |
71 | evFilter <- eventType
72 | evds <- dplyr::filter(.data=ds, .data$eventType==evFilter)
73 | events <- data.frame(dplyr::collect(evds))
74 |
75 | return(events)
76 |
77 | }
78 |
--------------------------------------------------------------------------------
/neonUtilities/R/chem_bundles.R:
--------------------------------------------------------------------------------
1 | #' Bundled chemistry data product information
2 | #'
3 | #' A dataset containing NEON data product codes of terrestrial chemistry data products and the "home" data products they are bundled with.
4 | #'
5 | #' @format A data frame with 2 variables:
6 | #' \describe{
7 | #' \item{product}{Data product ID of a terrestrial chemistry product}
8 | #' \item{homeProduct}{Data product ID of the corresponding home data product}
9 | #' }
10 | #' @source NEON data product bundles
11 | "chem_bundles"
12 |
--------------------------------------------------------------------------------
/neonUtilities/R/cleanUp.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Clean up folder after stacking
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Remove unzipped monthly data folders
9 | #'
10 | #' @keywords internal
11 | #' @param folder The file path to the folder that needs to be cleaned up (the root directory of the data package)
12 | #' @param orig The list of files that were present in the folder before unzipping and stacking
13 | #' @return Only the folders created during unzip will be deleted. All custom folders/files and the stackedFiles output folder will be retained.
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # Changelog and author contributions / copyrights
19 | # Christine Laney (2018-01-10)
20 | # Nathan Mietkiewicz (2020-02-12)
21 | ##############################################################################################
22 | cleanUp <- function(folder, orig) {
23 |
24 | currentFileList <- paste(folder, orig, sep="/")
25 |
26 | if(length(currentFileList) > 0) {
27 | unlink(currentFileList, recursive = TRUE)
28 | }
29 |
30 | writeLines("All unzipped monthly data folders have been removed.")
31 | }
32 |
--------------------------------------------------------------------------------
/neonUtilities/R/convByteSize.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Convert a number of bytes into megabytes or gigabytes
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' For any number of bytes, convert to a number of MB or GB
9 | #'
10 | #' @keywords internal
11 | #' @param objSize The size in bytes
12 | #' @return The size of the file in megabytes or gigabytes
13 | #'
14 | #' @references
15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
16 |
17 | #' @export
18 |
19 | # changelog and author contributions / copyrights
20 | # Claire Lunch (2020-09-24)
21 | ##############################################################################################
22 |
23 | convByteSize <- function(objSize){
24 | mbs <- objSize/1e6
25 | if(mbs < 1000) {
26 | return(paste(mbs, " MB", sep=""))
27 | } else {
28 | gbs <- objSize/1e9
29 | return(paste(gbs, " GB", sep=""))
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/neonUtilities/R/dateConvert.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Convert date stamps from character
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Attempt to convert date stamps from character, iterating through known NEON date formats
9 | #'
10 | #' @keywords internal
11 | #' @param dates A vector of date values in character format [character]
12 | #' @param useFasttime Should the fasttime package be used for date conversion? Defaults to false. [logical]
13 | #' @return A POSIXct vector, if possible; if conversion was unsuccessful, the original vector is returned
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # Changelog and author contributions / copyrights
19 | # Claire Lunch (2019-11-08)
20 | # Claire Lunch 2023-06-30: Add fasttime option
21 | ##############################################################################################
22 |
23 | dateConvert <- function(dates, useFasttime=FALSE){
24 |
25 | if(useFasttime) {
26 | d <- try(fasttime::fastPOSIXct(dates, tz='GMT'), silent=T)
27 | if(any(c(class(d)=='try-error', all(is.na(d))))) {
28 | d <- dates
29 | }
30 | } else {
31 | d <- try(as.POSIXct(dates, format='%Y-%m-%dT%H:%M:%S', tz='GMT'), silent=T)
32 | if(any(c(class(d)=='try-error', all(is.na(d))))) {
33 | d <- try(as.POSIXct(dates, format='%Y-%m-%dT%H:%M', tz='GMT'), silent=T)
34 | }
35 | if(any(c(class(d)=='try-error', all(is.na(d))))) {
36 | d <- try(as.POSIXct(dates, format='%Y-%m-%dT%H', tz='GMT'), silent=T)
37 | }
38 | if(any(c(class(d)=='try-error', all(is.na(d))))) {
39 | d <- try(as.POSIXct(dates, format='%Y-%m-%d', tz='GMT'), silent=T)
40 | }
41 | if(any(c(class(d)=='try-error', all(is.na(d))))) {
42 | d <- dates
43 | }
44 | }
45 | return(d)
46 | }
47 |
48 |
--------------------------------------------------------------------------------
/neonUtilities/R/eddyStampCheck.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Convert date stamps from character and check for only one record in a day
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Convert SAE time stamps to POSIX and check for missing data
9 | #'
10 | #' @keywords internal
11 | #' @param tab A table of SAE data
12 | #' @param useFasttime Should the fasttime package be used to convert time stamps?
13 | #' @return The same table of SAE data, with time stamps converted and empty records representing a single day (filler records inserted during processing) removed.
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # Changelog and author contributions / copyrights
19 | # Claire Lunch (2019-11-08)
20 | ##############################################################################################
21 |
22 | eddyStampCheck <- function(tab,
23 | useFasttime=FALSE){
24 |
25 | # convert time stamps
26 | tBgnErr <- FALSE
27 | tEndErr <- FALSE
28 |
29 | if(useFasttime) {
30 | tabBP <- try(fasttime::fastPOSIXct(tab$timeBgn, tz='GMT'), silent=T)
31 | tabEP <- try(fasttime::fastPOSIXct(tab$timeEnd, tz='GMT'), silent=T)
32 | } else {
33 | tabBP <- try(as.POSIXct(tab$timeBgn, format='%Y-%m-%dT%H:%M:%OS', tz='GMT'), silent=T)
34 | tabEP <- try(as.POSIXct(tab$timeEnd, format='%Y-%m-%dT%H:%M:%OS', tz='GMT'), silent=T)
35 | }
36 |
37 | if(any(c(inherits(tabBP,'try-error'), all(is.na(tabBP))))) {
38 | tBgnErr <- TRUE
39 | }
40 |
41 | if(any(c(inherits(tabBP,'try-error'), all(is.na(tabEP))))) {
42 | tEndErr <- TRUE
43 | }
44 |
45 | # if conversion failed, keep character time stamps and pass along message
46 | tabN <- tab
47 | err <- FALSE
48 | if(tBgnErr) {
49 | err <- TRUE
50 | } else {
51 | tabN$timeBgn <- tabBP
52 | }
53 |
54 | if(tEndErr) {
55 | err <- TRUE
56 | } else {
57 | tabN$timeEnd <- tabEP
58 | }
59 |
60 | # if conversion was successful, check for single-day empty records and remove
61 | if(err) {
62 | tabN <- tabN
63 | } else {
64 | dayDiff <- base::as.difftime(tabEP - tabBP)
65 | secDiff <- base::abs(base::as.numeric(dayDiff, units="secs"))
66 | dayDup <- which(secDiff >= 86399)
67 | if(length(dayDup)==0) {
68 | tabN <- tabN
69 | } else {
70 | tabN <- tabN[-dayDup,]
71 | }
72 | }
73 |
74 | return(list(tabN, err))
75 | }
76 |
77 |
--------------------------------------------------------------------------------
/neonUtilities/R/findDatatables.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Find data tables
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' List the names of the data tables within each folder
9 | #'
10 | #' @keywords internal
11 | #' @param folder The folder of the outputs
12 | #' @param fnames Full names - if true, then return the full file names including enclosing folders, if false, return only the file names
13 | #' @return a data frame of file names
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # Changelog and author contributions / copyrights
19 | # Christine Laney (2017-09-28)
20 | ##############################################################################################
21 |
22 | findDatatables <- function(folder, fnames = T){
23 | fs <- list.dirs(folder)
24 | g <- grep(pattern = "stackedFiles", x = fs)
25 | if(length(g) > 0){fs <- fs[-g]}
26 | fls <- character()
27 | for(i in 1:length(fs)){
28 | fls <- c(fls, list.files(fs[i], full.names = fnames))
29 | }
30 | return(fls[which(substr(fls, nchar(fls)-3, nchar(fls)) == ".csv")])
31 | }
32 |
--------------------------------------------------------------------------------
/neonUtilities/R/findTablesByFormat.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Find unique data tables in dataset
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Find the unique data tables that are present in the dataset (e.g., 2 minute vs 30 minute, or pinning vs identification data) and their types, based on the file name formatting. Adapted from findTablesUnique().
9 | #'
10 | #' @keywords internal
11 | #' @param datatables A list of data files
12 | #' @return An array of unique table names and their types
13 |
14 | #' @references
15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
16 |
17 | # Changelog and author contributions / copyrights
18 | # 2017-09-28 (Claire Lunch): created function, based on findTablesUnique()
19 | ##############################################################################################
20 |
21 | findTablesByFormat <- function(datatables){
22 | dt <- basename(datatables)
23 | splitNames <- strsplit(x = dt, split = "\\.")
24 | t <- character()
25 | for (i in 1:length(splitNames)){
26 | for(j in 3:length(splitNames[[i]])){
27 | s <- splitNames[[i]][j]
28 | s <- gsub(x = s, pattern = "_pub", replacement = "")
29 | if(s != "sensor_positions" & s != "science_review_flags" & length(grep('_', s, fixed=T))>0) {
30 | t <- c(t, s)
31 | }
32 | }
33 | }
34 |
35 | if(length(t)==0) {
36 | stop("No data tables found, only metadata. Try downloading expanded package, and check availability on the NEON data portal.")
37 | }
38 |
39 | tn <- unique(t)
40 | tt <- character(length(tn))
41 |
42 | for(k in 1:length(tn)) {
43 | names.k.all <- splitNames[union(grep(paste(".", tn[k], ".", sep=""), dt, fixed=T),
44 | grep(paste(".", tn[k], "_pub.", sep=""), dt, fixed=T))]
45 | tt.all <- lapply(names.k.all, function(xn) {
46 | if(length(which(xn==""))>0) {
47 | xn <- xn[-which(xn=="")]
48 | }
49 | if(length(xn)==5) {
50 | return("lab")
51 | } else {
52 | if(length(grep("[0-9]{4}-[0-9]{2}", xn))>0) {
53 | return("site-date")
54 | }
55 | else {
56 | return("site-all")
57 | }
58 | }
59 | })
60 | tt.temp <- unique(unlist(tt.all))
61 | if(length(tt.temp)>1) {
62 | stop(paste("In files to be stacked, table ", tn[k],
63 | " has been published under conflicting schedules. To avoid this problem, either work only with released data, or stack released and provisional data separately.",
64 | sep=""))
65 | }
66 | tt[k] <- tt.temp
67 | }
68 |
69 | tf <- cbind(tn,tt)
70 | tf <- data.frame(tf)
71 | names(tf) <- c("tableName","tableType")
72 |
73 | return(tf)
74 | }
75 |
--------------------------------------------------------------------------------
/neonUtilities/R/findTablesUnique.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Find unique data tables in dataset
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Find the unique data tables that are present in the dataset (e.g., 2 minute vs 30 minute, or pinning vs identification data)
9 | #'
10 | #' @keywords internal
11 | #' @param datatables A list of data files
12 | #' @return An array of unique table names
13 |
14 | #' @references
15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
16 |
17 | # Changelog and author contributions / copyrights
18 | # 2017-09-28 (Christine Laney): created original function
19 | # 2018-04-13 (Christine Laney): updated to check for extra "_pub" appending table names, and remove
20 | ##############################################################################################
21 |
22 | findTablesUnique <- function(datatables, tabletypes){
23 | dt <- datatables
24 | tt <- tabletypes
25 | splitNames <- strsplit(x = dt, split = "\\.")
26 | t <- character()
27 | for (i in 1:length(splitNames)){
28 | for(j in 1:length(splitNames[[i]])){
29 | s <- splitNames[[i]][j]
30 | s <- gsub(x = s, pattern = "_pub", replacement = "")
31 | if(s %in% tt$tableName){
32 | t <- c(t, s)
33 | }
34 | }
35 | if(length(splitNames[[i]]) == 4){
36 | n <- splitNames[[i]][3]
37 | n <- gsub(x = n, pattern = "_pub", replacement = "")
38 | t <- c(t, n)
39 | }
40 | if(length(splitNames[[i]]) %in% c(10,11)){
41 | n <- splitNames[[i]][7]
42 | n <- gsub(x = n, pattern = "_pub", replacement = "")
43 | t <- c(t, n)
44 | }
45 | if(length(splitNames[[i]]) == 12){
46 | n <- splitNames[[i]][11]
47 | n <- gsub(x = n, pattern = "_pub", replacement = "")
48 | t <- c(t, n)
49 | }
50 | if(length(splitNames[[i]]) == 14){
51 | n <- splitNames[[i]][10]
52 | n <- gsub(x = n, pattern = "_pub", replacement = "")
53 | t <- c(t, n)
54 | }
55 | }
56 | return(unique(t))
57 | }
58 |
--------------------------------------------------------------------------------
/neonUtilities/R/getAPI.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get the data from API
3 |
4 | #' @author
5 | #' Nate Mietkiewicz \email{mietkiewicz@battelleecology.org}
6 |
7 | #' @description Accesses the API with options to use the user-specific API token generated within data.neonscience.org user accounts.
8 | #'
9 | #' @keywords internal
10 | #' @param apiURL The API endpoint URL
11 | #' @param token User specific API token (generated within data.neonscience.org user accounts). Optional.
12 |
13 | #' @references
14 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
15 |
16 | # Changelog and author contributions / copyrights
17 | # 2020-05-21 (Claire Lunch): Modified to check for reaching rate limit
18 | # 2020-03-21 (Nate Mietkiewicz): Created original function
19 | ##############################################################################################
20 |
21 | getAPI <- function(apiURL, token=NA_character_){
22 |
23 | if(!curl::has_internet()) {
24 | message("No internet connection detected. Cannot access NEON API.")
25 | return(invisible())
26 | }
27 |
28 | if(identical(token, "")) {
29 | token <- NA_character_
30 | }
31 |
32 | usera <- paste("neonUtilities/", utils::packageVersion("neonUtilities"), " R/",
33 | R.Version()$major, ".", R.Version()$minor, " ", commandArgs()[1],
34 | " ", R.Version()$platform, sep="")
35 |
36 | if(is.na(token)) {
37 |
38 | # make 5 attempts to access - if rate limit is reached every time, give up
39 | j <- 1
40 | while(j < 6) {
41 |
42 | req <- try(httr::GET(apiURL, httr::user_agent(usera)), silent=T)
43 |
44 | # check for no response
45 | if(!inherits(req, "response")) {
46 | message("No response. NEON API may be unavailable, check NEON data portal for outage alerts. If the problem persists and can't be traced to an outage alert, check your computer for firewall or other security settings preventing R from accessing the internet.")
47 | return(invisible())
48 | }
49 |
50 | # if rate limit is reached, pause
51 | if(!is.null(req$headers$`x-ratelimit-limit`)) {
52 |
53 | if(req$headers$`x-ratelimit-remaining`<=1) {
54 | message(paste("Rate limit reached. Pausing for ",
55 | req$headers$`x-ratelimit-reset`,
56 | " seconds to reset.", sep=""))
57 | Sys.sleep(req$headers$`x-ratelimit-reset`)
58 | j <- j+1
59 | } else {
60 | j <- j+5
61 | }
62 | } else {
63 | j <- j+5
64 | }
65 | }
66 |
67 | } else {
68 |
69 | # same process as in non-token case: make 5 attempts
70 |
71 | j <- 1
72 | while(j < 6) {
73 |
74 | req <- try(httr::GET(apiURL, httr::user_agent(usera),
75 | httr::add_headers(.headers = c('X-API-Token'= token,
76 | 'accept' = 'application/json'))),
77 | silent=T)
78 |
79 | # check for no response
80 | if(!inherits(req, "response")) {
81 | message("No response. NEON API may be unavailable, check NEON data portal for outage alerts. If the problem persists and can't be traced to an outage alert, check your computer for firewall or other security settings preventing R from accessing the internet.")
82 | return(invisible())
83 | }
84 |
85 | # first check for null, since unlimited tokens don't have these headers
86 | if(!is.null(req$headers$`x-ratelimit-limit`)) {
87 |
88 | # if rate limit is reached, pause
89 | if(req$headers$`x-ratelimit-remaining`<=1) {
90 | message(paste("Rate limit reached. Pausing for ",
91 | req$headers$`x-ratelimit-reset`,
92 | " seconds to reset.", sep=""))
93 | Sys.sleep(req$headers$`x-ratelimit-reset`)
94 | j <- j+1
95 | } else {
96 | j <- j+5
97 | }
98 | } else {
99 | j <- j+5
100 | }
101 | }
102 | }
103 |
104 | return(req)
105 |
106 | }
107 |
--------------------------------------------------------------------------------
/neonUtilities/R/getAPIHeaders.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get only headers from API
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description Accesses the API with options to use the user-specific API token generated within neon.datascience user accounts.
8 | #'
9 | #' @keywords internal
10 | #' @param apiURL The API endpoint URL
11 | #' @param token User specific API token (generated within data.neonscience.org user accounts). Optional.
12 |
13 | #' @references
14 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
15 |
16 | # Changelog and author contributions / copyrights
17 | # 2021-01-24 (Claire Lunch): Adapted from getAPI()
18 | ##############################################################################################
19 |
20 | getAPIHeaders <- function(apiURL, token=NA_character_){
21 |
22 | if(!curl::has_internet()) {
23 | message("No internet connection detected. Cannot access NEON API.")
24 | return(invisible())
25 | }
26 |
27 | if(identical(token, "")) {
28 | token <- NA_character_
29 | }
30 |
31 | usera <- paste("neonUtilities/", utils::packageVersion("neonUtilities"), " R/",
32 | R.Version()$major, ".", R.Version()$minor, " ", commandArgs()[1],
33 | " ", R.Version()$platform, sep="")
34 |
35 | if(is.na(token)) {
36 |
37 | # make 5 attempts to access - if rate limit is reached every time, give up
38 | j <- 1
39 | while(j < 6) {
40 |
41 | req <- try(httr::HEAD(apiURL, httr::user_agent(usera)), silent=T)
42 |
43 | # check for no response
44 | if(!inherits(req, "response")) {
45 | message("No response. NEON API may be unavailable, check NEON data portal for outage alerts.")
46 | return(invisible())
47 | }
48 |
49 | # if rate limit is reached, pause
50 | if(!is.null(req$headers$`x-ratelimit-limit`)) {
51 |
52 | if(req$headers$`x-ratelimit-remaining`<=1) {
53 | message(paste("Rate limit reached. Pausing for ",
54 | req$headers$`x-ratelimit-reset`,
55 | " seconds to reset.", sep=""))
56 | Sys.sleep(req$headers$`x-ratelimit-reset`)
57 | j <- j+1
58 | } else {
59 | j <- j+5
60 | }
61 | } else {
62 | j <- j+5
63 | }
64 | }
65 |
66 | } else {
67 |
68 | # same process as in non-token case: make 5 attempts
69 |
70 | j <- 1
71 | while(j < 6) {
72 |
73 | req <- try(httr::HEAD(apiURL, httr::user_agent(usera),
74 | httr::add_headers(.headers = c('X-API-Token'= token,
75 | 'accept' = 'application/json'))),
76 | silent=T)
77 |
78 | # check for no response
79 | if(!inherits(req, "response")) {
80 | message("No response. NEON API may be unavailable, check NEON data portal for outage alerts.")
81 | return(invisible())
82 | }
83 |
84 | # first check for null, since unlimited tokens don't have these headers
85 | if(!is.null(req$headers$`x-ratelimit-limit`)) {
86 |
87 | # if rate limit is reached, pause
88 | if(req$headers$`x-ratelimit-remaining`<=1) {
89 | message(paste("Rate limit reached. Pausing for ",
90 | req$headers$`x-ratelimit-reset`,
91 | " seconds to reset.", sep=""))
92 | Sys.sleep(req$headers$`x-ratelimit-reset`)
93 | j <- j+1
94 | } else {
95 | j <- j+5
96 | }
97 | } else {
98 | j <- j+5
99 | }
100 | }
101 | }
102 |
103 | return(req)
104 |
105 | }
106 |
--------------------------------------------------------------------------------
/neonUtilities/R/getAttributes.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Extract attributes from eddy covariance H5 files
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Extract attribute metadata from H5 files
9 | #'
10 | #' @param fil File path to the H5 file to extract attributes from [character]
11 | #' @param sit The site, for site attributes. Must match site of file path. [character]
12 | #' @param type The type of attributes to retrieve. [character]
13 | #' @param valName If CO2 validation metadata are requested, the H5 name of the level where they can be found. [character]
14 |
15 | #' @keywords internal
16 | #' @return A data frame with one row containing the extracted attributes
17 |
18 | #' @references
19 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 |
21 | # changelog and author contributions / copyrights
22 | # Claire Lunch (2023-03-21)
23 | ##############################################################################################
24 |
25 | getAttributes <- function(fil, sit, type, valName) {
26 |
27 | if(grepl("basic", fil)) {
28 | mnth <- regmatches(fil, regexpr("20[0-9]{2}-[0-9]{2}", fil))
29 | } else {
30 | mnth <- regmatches(fil, regexpr("20[0-9]{2}-[0-9]{2}-[0-9]{2}", fil))
31 | }
32 |
33 | if(type=="site") {
34 | lev <- sit
35 | }
36 | if(type=="root") {
37 | lev <- "/"
38 | }
39 | if(type=="val") {
40 | lev <- valName
41 | }
42 |
43 | gAttr <- base::try(rhdf5::h5readAttributes(fil, name=lev), silent=T)
44 |
45 | if(inherits(gAttr, "try-error")) {
46 | gAttr <- data.frame(matrix(data=c(sit, mnth), ncol=2, nrow=1))
47 | names(gAttr) <- c("site","date")
48 | } else {
49 | if(any(lapply(gAttr, length)>1)) {
50 | for(i in which(lapply(gAttr, length)>1)) {
51 | gAttr[i] <- paste0(gAttr[i], collapse=",")
52 | }
53 | }
54 | gAttr <- c(sit, mnth, gAttr)
55 | names(gAttr)[1:2] <- c("site", "date")
56 | }
57 | return(gAttr)
58 | }
59 |
60 |
--------------------------------------------------------------------------------
/neonUtilities/R/getAvg.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get a list of the available averaging intervals for a data product
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Most IS products are available at multiple averaging intervals; get a list of what's available for a given data product
9 | #'
10 | #' @param dpID The identifier of the NEON data product, in the form DPL.PRNUM.REV, e.g. DP1.00006.001
11 | #' @param token User specific API token (generated within data.neonscience.org user accounts)
12 |
13 | #' @return A vector of the available averaging intervals, typically in minutes.
14 |
15 | #' @examples
16 | #' # Get available averaging intervals for PAR data product
17 | #' getAvg("DP1.00024.001")
18 |
19 | #' @references
20 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
21 |
22 | #' @export
23 |
24 | # changelog and author contributions / copyrights
25 | # Claire Lunch (2018-05-11)
26 | # original creation
27 | ##############################################################################################
28 |
29 | getAvg <- function(dpID, token = NA_character_) {
30 |
31 | message("getAvg() is deprecated; use getTimeIndex() in future", call.=FALSE)
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/neonUtilities/R/getCitation.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get a Bibtex citation for NEON data with a DOI, or generate a provisional Bibtex citation
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Use the DOI Foundation API to get Bibtex-formatted citations for NEON data, or use a template to generate a Bibtex citation for provisional data. Helper function to download and stacking functions.
9 |
10 | #' @param dpID The data product ID of the data to be cited [character]
11 | #' @param release The data release to be cited. Can be provisional. [character]
12 |
13 | #' @return A character string containing the Bibtex citation
14 | #'
15 | #' @export
16 |
17 | #' @examples
18 | #' \dontrun{
19 | #' # Get the citation for Breeding landbird point counts (DP1.10003.001), RELEASE-2023
20 | #' cit <- getCitation(dpID="DP1.10003.001", release="RELEASE-2023")
21 | #' }
22 |
23 | #' @references
24 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
25 |
26 | # Changelog and author contributions / copyrights
27 | # Claire Lunch (2023-04-18)
28 | ##############################################################################################
29 |
30 | getCitation <- function(dpID=NA_character_, release=NA_character_) {
31 |
32 | if(is.na(dpID) | is.na(release)) {
33 | stop("dpID and release are required inputs.")
34 | }
35 |
36 | # if release=PROVISIONAL, construct provisional citation from template
37 | if(release=="PROVISIONAL") {
38 |
39 | cit <- cit_prov_template
40 | cit <- base::gsub(pattern="DPID", replacement=dpID, x=cit)
41 | cit <- base::gsub(pattern="YEAR", replacement=format(Sys.Date(), "%Y"), x=cit)
42 |
43 | # get product name from NEON API
44 | req <- httr::GET(paste("https://data.neonscience.org/api/v0/products/",
45 | dpID, sep=""))
46 | if(is.null(req)) {return(invisible())}
47 | if(!inherits(req, "response")) {return(invisible())}
48 |
49 | df <- jsonlite::fromJSON(httr::content(req, as="text", encoding="UTF-8"),
50 | simplifyDataFrame=TRUE, flatten=TRUE)
51 | nm <- df$data$productName
52 | cit <- base::gsub(pattern="NAME", replacement=nm, x=cit)
53 |
54 | } else {
55 |
56 | doi <- try(getNeonDOI(dpID=dpID, release=release), silent=TRUE)
57 | if(!inherits(doi, "try-error")) {
58 |
59 | req <- httr::GET(paste("https://doi.org/", doi$DOI, sep=""),
60 | httr::accept("application/x-bibtex"))
61 |
62 | # check for no or empty response
63 | if(is.null(req)) {
64 | message("No response. DOI Foundation API may be unavailable, check for outage alerts.")
65 | return(invisible())
66 | }
67 |
68 | if(!inherits(req, "response")) {
69 | message("No response. DOI Foundation API may be unavailable, check for outage alerts.")
70 | return(invisible())
71 | }
72 |
73 | # read response
74 | cit <- httr::content(req, as="text", encoding="UTF-8")
75 |
76 | } else {
77 | message(paste("DOI not found for data product ID ", dpID, "and release",
78 | release, ". Check inputs.", sep=""))
79 | return(invisible())
80 | }
81 |
82 | }
83 |
84 | return(cit)
85 |
86 | }
87 |
--------------------------------------------------------------------------------
/neonUtilities/R/getDatatable.R:
--------------------------------------------------------------------------------
1 | #' Get NEON data table
2 | #'
3 | #'
4 | #' @author Eric R. Sokol \email{esokol@battelleecology.org}
5 | #'
6 | #'
7 | #' @description This is a function to retrieve a data table
8 | #' from the NEON data portal for sites and dates provided by the
9 | #' enduser. NOTE that this function only works for NEON
10 | #' Observation System (OS) data products, and only for select tables
11 | #'
12 | #' @param dpid character sting for NEON data product ID
13 | #' @param data_table_name character sting for name of the data table to download, e.g., 'sls_soilCoreCollection'
14 | #' @param sample_location_list list of sites, domains, etc. If NA, retrieve all data for the given data table / dpid combination.
15 | #' @param sample_location_type character sting for location type, such as 'siteID'. Must be one of the NEON controlled terms. If you're unsure, use 'siteID'
16 | #' @param sample_date_min start date for query. Default is 1-Jan-2012, and this should capture the earliest NEON data record.
17 | #' @param sample_date_max end date for query. Default is current date.
18 | #' @param sample_date_format date format. Default/expected format is yyyy-mm-dd
19 | #' @param data_package_type package type, either 'basic' or 'expanded'. If unsure, use 'expanded'
20 | #' @param url_prefix_data data endpoint for NEON API.
21 | #' @param url_prefix_products products endpoint for NEON API.
22 | #' @param token User specific API token (generated within data.neonscience.org user accounts)
23 | #'
24 | #' @return data frame with selected NEON data
25 | #'
26 | #'
27 | #' @references License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
28 | #'
29 | #'
30 | #' @export
31 | getDatatable <- function(
32 | dpid = NA, #data product ID
33 | data_table_name = NA, #data table name
34 | sample_location_list = NA, # list of sites, domains, etc.
35 | sample_location_type = 'siteID', #location type
36 | sample_date_min = '2012-01-01', #start date -- default is 1-Jan-2012
37 | sample_date_max = Sys.Date(), #end date -- default is current date
38 | sample_date_format = '%Y-%m-%d', #date format
39 | ### more defaults
40 | data_package_type = 'basic',
41 | url_prefix_data = 'https://data.neonscience.org/api/v0/data/',
42 | url_prefix_products = 'https://data.neonscience.org/api/v0/products/',
43 | token = NA_character_){
44 |
45 | message("getDatatable() is deprecated. Use loadByProduct() with input parameter tabl=")
46 | return(invisible())
47 |
48 | }
49 |
50 |
--------------------------------------------------------------------------------
/neonUtilities/R/getEddyLog.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get the full issue log set for the SAE bundle
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Use the NEON API to get the issue log from all products in the bundle in a user-friendly format
9 |
10 | #' @param token User specific API token (generated within data.neonscience.org user accounts)
11 | #' @keywords internal
12 | #'
13 | #' @return A table of issues reported for the data product.
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # Changelog and author contributions / copyrights
19 | # Claire Lunch (2021-10-27)
20 | ##############################################################################################
21 | getEddyLog <- function(token=NA_character_) {
22 |
23 | bundleDPs <- c("DP1.00007.001", "DP1.00010.001", "DP1.00034.001", "DP1.00035.001",
24 | "DP1.00036.001", "DP1.00037.001", "DP1.00099.001", "DP1.00100.001",
25 | "DP2.00008.001", "DP2.00009.001", "DP2.00024.001", "DP3.00008.001",
26 | "DP3.00009.001", "DP3.00010.001", "DP4.00002.001", "DP4.00007.001",
27 | "DP4.00067.001", "DP4.00137.001", "DP4.00201.001", "DP4.00200.001")
28 |
29 | eddyLog <- character()
30 | for(i in bundleDPs) {
31 |
32 | req <- getAPI(apiURL = paste0("https://data.neonscience.org/api/v0/products/", i),
33 | token = token)
34 |
35 | if(is.null(req)) {
36 | return(invisible())
37 | }
38 |
39 | allproductinfo <- jsonlite::fromJSON(httr::content(req, as="text", encoding="UTF-8"),
40 | simplifyDataFrame=TRUE, flatten=TRUE)
41 |
42 | productlog <- allproductinfo[["data"]]$changeLogs
43 |
44 | if(!is.null(productlog)) {
45 | if(nrow(productlog)>0) {
46 | dataSubproduct <- rep(i, nrow(productlog))
47 | productlog <- cbind(dataSubproduct, productlog)
48 | eddyLog <- rbind(eddyLog, productlog)
49 | }
50 | }
51 |
52 | }
53 | return(eddyLog)
54 | }
55 |
--------------------------------------------------------------------------------
/neonUtilities/R/getFileUrls.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 | #' Christine Laney \email{claney@battelleecology.org}
7 |
8 | #' @description Used to generate a data frame of available AOP files.
9 | #'
10 | #' @param m.urls The monthly API URL for the AOP files
11 | #' @param include.provisional T or F, should provisional data be included in downloaded files?
12 | #' @param token User specific API token (generated within data.neonscience.org user accounts)
13 | #'
14 | #' @keywords internal
15 |
16 | #' @return A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0)
17 |
18 | #' @references
19 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 |
21 | # Changelog and author contributions / copyrights
22 | # Claire Lunch (2018-02-19): original creation
23 | # Christine Laney (2018-03-05): Added functionality to get new list of URLs if the old ones expire, during the download stream.
24 |
25 | ##############################################################################################
26 |
27 | # get and stash the file names, S3 URLs, file size, and download status (default = 0) in a data frame
28 | getFileUrls <- function(m.urls, include.provisional, token=NA){
29 | url.messages <- character()
30 | file.urls <- c(NA, NA, NA)
31 | releases <- character()
32 | prov.message <- "no"
33 | for(i in 1:length(m.urls)) {
34 |
35 | tmp <- getAPI(apiURL = m.urls[i], token = token)
36 |
37 | if(tmp$status_code!=200) {
38 | message(paste("Data file retrieval failed with code ", tmp$status_code,
39 | ". Check NEON data portal for outage alerts.", sep=""))
40 | return(invisible())
41 | }
42 |
43 | tmp.files <- jsonlite::fromJSON(httr::content(tmp, as='text', encoding='UTF-8'),
44 | simplifyDataFrame=TRUE, flatten=TRUE)
45 |
46 | # check for no files
47 | if(length(tmp.files$data$files)==0) {
48 | message(paste("No files found for site", tmp.files$data$siteCode,
49 | "and year", tmp.files$data$month, sep=" "))
50 | next
51 | }
52 |
53 | # get release info
54 | if(isFALSE(include.provisional) & tmp.files$data$release=="PROVISIONAL") {
55 | file.urls <- file.urls
56 | releases <- releases
57 | prov.message <- "yes"
58 | } else {
59 | releases <- c(releases, tmp.files$data$release)
60 |
61 | file.urls <- rbind(file.urls, cbind(tmp.files$data$files$name,
62 | tmp.files$data$files$url,
63 | tmp.files$data$files$size))
64 | }
65 | }
66 |
67 | if(all(is.na(file.urls)) & prov.message=="yes") {
68 | stop("All files found were provisional. Modify download query (dates and/or sites) or, if you want to use provisional data, use input parameter include.provisional=TRUE.")
69 | }
70 |
71 | if(prov.message=="yes") {
72 | message("Provisional data were excluded from available files list. To download provisional data, use input parameter include.provisional=TRUE.")
73 | }
74 |
75 | # get size info
76 | file.urls <- data.frame(file.urls, row.names=NULL)
77 | colnames(file.urls) <- c("name", "URL", "size")
78 | file.urls$URL <- as.character(file.urls$URL)
79 | file.urls$name <- as.character(file.urls$name)
80 |
81 | if(length(url.messages) > 0){writeLines(url.messages)}
82 | file.urls <- file.urls[-1,]
83 | release <- unique(releases)
84 | return(list(file.urls, release))
85 |
86 | }
87 |
--------------------------------------------------------------------------------
/neonUtilities/R/getIssueLog.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get the issue log for a specific data product
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Use the NEON API to get the issue log in a user-friendly format
9 |
10 | #' @param dpID The data product identifier, formatted as DP#.#####.###
11 | #' @param token User specific API token (generated within data.neonscience.org user accounts)
12 |
13 | #' @return A table of issues reported for the data product.
14 |
15 | #' @examples
16 | #' # Get documentation and availability of plant foliar properties data product
17 | #' cfcIssues <- getIssueLog("DP1.10026.001")
18 |
19 | #' @export
20 |
21 | #' @references
22 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
23 |
24 | # Changelog and author contributions / copyrights
25 | # Claire Lunch (2021-10-26)
26 | ##############################################################################################
27 | getIssueLog <- function(dpID=NA, token=NA_character_) {
28 |
29 | # error message if dpID isn't formatted as expected
30 | if(regexpr("DP[1-4]{1}.[0-9]{5}.00[0-9]{1}",dpID)[1]!=1) {
31 | stop(paste(dpID, "is not a properly formatted data product ID. The correct format is DP#.#####.00#", sep=" "))
32 | }
33 |
34 | if(dpID=="DP4.00200.001") {
35 | issuelog <- getEddyLog(token=token)
36 | } else {
37 | req <- getAPI(apiURL = paste0("https://data.neonscience.org/api/v0/products/", dpID),
38 | token = token)
39 |
40 | if(is.null(req)) {
41 | return(invisible())
42 | }
43 |
44 | allproductinfo <- jsonlite::fromJSON(httr::content(req, as="text", encoding="UTF-8"),
45 | simplifyDataFrame=TRUE, flatten=TRUE)
46 |
47 | issuelog <- allproductinfo[["data"]]$changeLogs
48 | }
49 |
50 | return(issuelog)
51 | }
52 |
--------------------------------------------------------------------------------
/neonUtilities/R/getPackage.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get NEON data package
3 | #'
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Get a zipped file for a single data product, site, and year-month combination. Use the NEON data portal or API to determine data
9 | #' availability by data product, site, and year-month combinations.
10 | #'
11 | #' @param dpID The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001
12 | #' @param site_code A four-letter NEON research site code, such as HEAL for Healy.
13 | #' @param year_month The year and month of interest, in format YYYY-MM.
14 | #' @param package Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.
15 | #' @param savepath The location to save the output files to
16 | #' @return A zipped monthly file
17 |
18 | #' @references
19 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 |
21 | #' @export
22 |
23 | # Changelog and author contributions / copyrights
24 | # 2018-01-09 (Christine Laney): Created function
25 | # 2018-04-10 (Christine Laney): Exported function and added parameter for where to save the downloaded file.
26 | # 2022-02-07 (Claire Lunch): Original function relied on zip files, which are no longer stored. Redirected to zipsByProduct(), as first step to deprecating.
27 | ##############################################################################################
28 |
29 | getPackage <- function(dpID, site_code, year_month, package="basic", savepath = getwd()){
30 |
31 | warning("getPackage() will be deprecated soon, use zipsByProduct(). For back-compatibility, your request has been sent to zipsByProduct().",
32 | call. = FALSE)
33 | zipsByProduct(dpID=dpID, site=site_code, startdate=year_month,
34 | enddate=year_month, package=package,
35 | savepath=savepath, check.size=FALSE)
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/neonUtilities/R/getProductInfo.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get NEON data product information
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Use the NEON API to get data product information such as availability, science team, etc.
9 |
10 | #' @importFrom jsonlite fromJSON
11 | #' @importFrom httr GET
12 | #' @param dpID The data product id (optional), formated as DP#.#####.###
13 | #' @param token User specific API token (generated within data.neonscience.org user accounts)
14 |
15 | #' @return A named list of metadata and availability information for a single data product. If the dpID argument is omitted, a table of information for all data products in the NEON catalog.
16 |
17 | #' @examples
18 | #' # Get documentation and availability of plant foliar properties data product
19 | #' cfcInfo <- getProductInfo("DP1.10026.001")
20 |
21 | #' @export
22 |
23 | #' @references
24 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
25 |
26 | # Changelog and author contributions / copyrights
27 | # Christine Laney (2017-10-01)
28 | ##############################################################################################
29 | getProductInfo <- function(dpID="", token = NA){
30 |
31 | req <- getAPI(apiURL = paste0("https://data.neonscience.org/api/v0/products/", dpID),
32 | token = token)
33 |
34 | if(is.null(req)) {
35 | return(invisible())
36 | }
37 |
38 | avail <- jsonlite::fromJSON(httr::content(req, as="text", encoding="UTF-8"),
39 | simplifyDataFrame=TRUE, flatten=TRUE)
40 |
41 | avail <- avail[["data"]]
42 | return(avail)
43 | }
44 |
45 | # query the products endpoint for the product requested
46 |
--------------------------------------------------------------------------------
/neonUtilities/R/getProductSensors.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get data product-sensor relationships
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Pull all data from the NEON API /products endpoint, create a data frame with data product ID, data product name, and sensor type.
9 |
10 | #' @return A data frame
11 |
12 | #' @examples
13 | #' \dontrun{
14 | #' sensors <- getProductSensors()
15 | #' }
16 |
17 | #' @export
18 |
19 |
20 | #' @references
21 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 |
23 | # Changelog and author contributions / copyrights
24 | # 2018-04-24 Christine Laney: Created function
25 | ##############################################################################################
26 |
27 | getProductSensors <- function(){
28 | products <- jsonlite::fromJSON("https://data.neonscience.org/api/v0/products", flatten=T)
29 | products <- products[["data"]]
30 | df <- data.frame(dpID = NA, dpName = NA, sensor = NA)
31 | for(i in 1:nrow(products)){
32 | dpID <- products$productCode[i]
33 | dpName <- products$productName[i]
34 | sensor <- products$productSensor[i]
35 | df <- rbind(df, c(dpID, dpName, sensor))
36 | }
37 | df <- df[-1, ]
38 | return(df)
39 | }
40 |
41 |
--------------------------------------------------------------------------------
/neonUtilities/R/getReadmePublicationDate.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Scrape the publication date from each ReadMe file
3 |
4 | #' @author
5 | #' Nathan Mietkiewicz \email{mietkiewicz@battelleecology.org}
6 |
7 | #' @description
8 | #' Given a directory, this will recursively list all of the ReadMe files that were unzipped.
9 | #' This should result in a single text file with a list of all of the publication dates from the ReadMe file.
10 |
11 | #' @param savepath The root folder directory where the ReadMe files are located.
12 | #' @param out_filepath The output directory and filename.
13 | #' @param dpID The data product identifier
14 | #'
15 | #' @keywords internal
16 |
17 | #' @references
18 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
19 |
20 | # Changelog and author contributions / copyrights
21 | # 2019-10-08 (Nathan Mietkiewicz): Original creation
22 |
23 | ##############################################################################################
24 | getReadmePublicationDate <- function(savepath, out_filepath, dpID) {
25 |
26 | dpnum <- substring(dpID, 5, 9)
27 | out_filepath_name <- paste0(out_filepath, '/readme_', dpnum, '.txt')
28 |
29 | if(file.exists(out_filepath_name)) {
30 | unlink(out_filepath_name)
31 | }
32 | #writeLines("Stacking ReadMe documentation")
33 | readme_list <- list.files(savepath, pattern = '.readme.',
34 | recursive = TRUE, full.names = TRUE)
35 | if(length(readme_list)==0) {
36 | message("No readme file found")
37 | } else {
38 |
39 | readme_recent <- getRecentPublication(readme_list)[[1]]
40 | txt_file <- utils::read.delim(readme_recent, header=FALSE, quote="", stringsAsFactors=FALSE)
41 | txt_file <- txt_file$V1[grep("Date-Time", txt_file$V1, invert=TRUE)]
42 |
43 | tables <- table_types[which(table_types$productID==dpID),]
44 | if(nrow(tables)>0) {
45 | qInd <- grep('QUERY', txt_file)
46 | dPackInd <- grep('CONTENTS', txt_file)
47 | downPackInd <- grep('Basic download package', txt_file)
48 |
49 | txt_file[I(dPackInd+3)] <- paste('This data product contains up to', nrow(tables), 'data tables:')
50 | txt_file[I(dPackInd+5):I(dPackInd+4+nrow(tables))] <- paste(tables$tableName, tables$tableDesc, sep=' - ')
51 | txt_file[I(dPackInd+5+nrow(tables))] <- 'If data are unavailable for the particular sites and dates queried, some tables may be absent.'
52 | txt_file <- txt_file[-c(qInd:I(dPackInd-2), I(dPackInd+6+nrow(tables)):I(downPackInd-1))]
53 | }
54 |
55 | cat("###################################\n", file = out_filepath_name)
56 | cat("########### Disclaimer ############\n", file = out_filepath_name, append=TRUE)
57 | cat('This is the most recent readme publication based on all site-date combinations used during stackByTable.\nInformation specific to the query, including sites and dates, has been removed. The remaining content reflects general metadata for the data product.\n', file = out_filepath_name, append=TRUE)
58 | cat("##################################\n", file = out_filepath_name, append=TRUE)
59 | cat("\n", file = out_filepath_name, append=TRUE)
60 | utils::write.table(txt_file, out_filepath_name, append=TRUE,
61 | row.names=FALSE, col.names=FALSE, quote=FALSE)
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/neonUtilities/R/getRecentPublication.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Returns the most recent files for those that do not need stacking
3 |
4 | #' @author
5 | #' Nathan Mietkiewicz \email{mietkiewicz@battelleecology.org}
6 |
7 | #' @description
8 | #' Given a list of files, this will order and return the file with the most recent publication date.
9 |
10 | #' @param inList The list of files.
11 | #'
12 | #' @keywords internal
13 |
14 | #' @references
15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
16 |
17 | # Changelog and author contributions / copyrights
18 | # 2019-10-08 (Nathan Mietkiewicz): Original creation
19 | # 2020-11-05 (Claire Lunch): Modify for compatibility with stackFromStore()
20 |
21 | ##############################################################################################
22 | getRecentPublication <- function(inList) {
23 |
24 | # if file dates are available, use file dates
25 | file_dates <- lapply(basename(inList), function(x) {
26 | regmatches(x, regexpr("[0-9]{8}T[0-9]{6}Z", x))
27 | })
28 | # if dates can't be found, move to dates in file path - for release 2021, these are all identical
29 | if(length(unlist(file_dates))==0) {
30 |
31 | path_dates <- lapply(inList, function(x) {
32 | regmatches(x, regexpr("[0-9]{8}T[0-9]{6}Z", x))
33 | })
34 | # if dates can't be found, use "max" of input file names. This should be very rare.
35 | if(length(unlist(path_dates))==0) {
36 | file_dates <- inList
37 | } else {
38 | file_dates <- path_dates
39 | }
40 |
41 | }
42 |
43 | outList <- inList[grep(max(unlist(file_dates)), inList)][1]
44 | return(list(outList, ifelse(nchar(file_dates[[1]])==16, max(unlist(file_dates)),
45 | "undetermined")))
46 | }
--------------------------------------------------------------------------------
/neonUtilities/R/getTaxonTable.R:
--------------------------------------------------------------------------------
1 | #' Get NEON taxon table
2 | #'
3 | #'
4 | #' @author Eric R. Sokol \email{esokol@battelleecology.org}
5 | #'
6 | #'
7 | #' @description This is a function to retrieve a taxon table
8 | #' from the NEON data portal for the taxon type by the
9 | #' enduser.
10 | #'
11 | #' @import httr jsonlite
12 | #'
13 | #' @param taxonType Character string for the taxonTypeCode. Must be one of ALGAE, BEETLE, BIRD,
14 | #' FISH, HERPETOLOGY, MACROINVERTEBRATE, MOSQUITO, MOSQUITO_PATHOGENS, SMALL_MAMMAL, PLANT, TICK
15 | #' @param recordReturnLimit Integer. The number of items to limit the result set to. If NA, will return all records in table.
16 | #' @param stream Character string, true or false. Option to obtain the result as a stream. Utilize for large requests.
17 | #' @param token User specific API token (generated within data.neonscience.org user accounts)
18 | #'
19 | #' @return data frame with selected NEON data
20 | #'
21 | #'
22 | #' @references License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
23 | #'
24 | #'
25 | #' @export
26 |
27 | getTaxonTable <- function(
28 | taxonType = NA, #string, taxonTypeCode, one of ALGAE, BEETLE, BIRD, FISH, HERPETOLOY, MACROINVERTEBRATE, MOSQUITO, MOSQUITO_PATHOGENS, SMALL_MAMMAL, PLANT, TICK
29 | recordReturnLimit = NA, #integer, The number of items to limit the result set to. If NA, will return all records in table.
30 | stream = 'true', #string, Option to obtain the result as a stream. Utilize for large requests.
31 | token = NA
32 | ){
33 |
34 | message("getTaxonTable() is deprecated in neonUtilities. Use the updated version of this function, renamed to getTaxonList(), in the neonOS package.")
35 |
36 | }
37 |
--------------------------------------------------------------------------------
/neonUtilities/R/getTileUrls.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 | #' Christine Laney \email{claney@battelleecology.org}
7 |
8 | #' @description Produces a data frame that is populated by available tiles for the AOP product.
9 | #'
10 | #' @param m.urls The monthly API URL for the AOP tile.
11 | #' @param tileEasting A vector containing the easting UTM coordinates of the locations to download.
12 | #' @param tileNorthing A vector containing the northing UTM coordinates of the locations to download.
13 | #' @param include.provisional T or F, should provisional data be included in downloaded files?
14 | #' @param token User specific API token (generated within data.neonscience.org user accounts). Optional.
15 | #'
16 | #' @keywords internal
17 | #'
18 | #' @return A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0)
19 |
20 | #' @references
21 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 |
23 | # Changelog and author contributions / copyrights
24 | # Claire Lunch (2018-02-19): original creation
25 | # Christine Laney (2018-03-05): Added functionality to get new list of URLs if the old ones expire, during the download stream.
26 |
27 | ##############################################################################################
28 |
29 | # get and stash the file names, S3 URLs, file size, and download status (default = 0) in a data frame
30 | getTileUrls <- function(m.urls, tileEasting, tileNorthing,
31 | include.provisional, token=NA_character_){
32 |
33 | url.messages <- character()
34 | file.urls <- c(NA, NA, NA)
35 | releases <- character()
36 | prov.message <- "no"
37 | for(i in 1:length(m.urls)) {
38 | tmp <- getAPI(m.urls[i], token=token)
39 | if(tmp$status_code!=200) {
40 | message(paste("Data file retrieval failed with code ", tmp$status_code,
41 | ". Check NEON data portal for outage alerts.", sep=""))
42 | return(invisible())
43 | }
44 |
45 | tmp.files <- jsonlite::fromJSON(httr::content(tmp, as="text", encoding="UTF-8"),
46 | simplifyDataFrame=T, flatten=T)
47 |
48 | # check for no files
49 | if(length(tmp.files$data$files)==0) {
50 | message(paste("No files found for site", tmp.files$data$siteCode,
51 | "and year", tmp.files$data$month, sep=" "))
52 | next
53 | }
54 |
55 | # get release info
56 | if(isFALSE(include.provisional) & tmp.files$data$release=="PROVISIONAL") {
57 | file.urls <- file.urls
58 | releases <- releases
59 | prov.message <- "yes"
60 | } else {
61 | releases <- c(releases, tmp.files$data$release)
62 |
63 | # filter to only files for the relevant tiles
64 | ind <- numeric()
65 | for(j in 1:length(tileEasting)) {
66 | ind.j <- intersect(grep(paste('_', tileEasting[j], '_', sep=''), tmp.files$data$files$name),
67 | grep(paste('_', tileNorthing[j], '_', sep=''), tmp.files$data$files$name))
68 | if(length(ind.j)>0) {
69 | ind <- c(ind, ind.j)
70 | } else {
71 | message(paste("No tiles found for easting ",
72 | tileEasting[j], " and northing ",
73 | tileNorthing[j], sep=""))
74 | }
75 | }
76 | ind <- unique(ind)
77 | tile.files <- tmp.files$data$files[ind,]
78 |
79 | file.urls <- rbind(file.urls, cbind(tile.files$name,
80 | tile.files$url,
81 | tile.files$size))
82 | }
83 |
84 | }
85 |
86 | if(all(is.na(file.urls)) & prov.message=="yes") {
87 | stop("All files found were provisional. Modify download query (dates and/or sites) or, if you want to use provisional data, use input parameter include.provisional=TRUE.")
88 | }
89 |
90 | if(prov.message=="yes") {
91 | message("Provisional data were excluded from available files list. To download provisional data, use input parameter include.provisional=TRUE.")
92 | }
93 |
94 | # get size info
95 | file.urls <- data.frame(file.urls, row.names=NULL)
96 | colnames(file.urls) <- c("name", "URL", "size")
97 | file.urls$URL <- as.character(file.urls$URL)
98 | file.urls$name <- as.character(file.urls$name)
99 |
100 | if(length(url.messages) > 0){writeLines(url.messages)}
101 | file.urls <- file.urls[-1,]
102 | release <- unique(releases)
103 | return(list(file.urls, release))
104 |
105 | }
106 |
--------------------------------------------------------------------------------
/neonUtilities/R/getTimeIndex.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get a list of the available time intervals for a data product
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Most IS products are available at multiple time intervals; get a list of what's available for a given data product
9 | #'
10 | #' @param dpID The identifier of the NEON data product, in the form DPL.PRNUM.REV, e.g. DP1.00006.001
11 | #' @param token User specific API token (generated within data.neonscience.org user accounts)
12 |
13 | #' @return A vector of the available time intervals, typically in minutes.
14 |
15 | #' @examples
16 | #' # Get available time intervals for PAR data product
17 | #' getTimeIndex("DP1.00024.001")
18 |
19 | #' @references
20 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
21 |
22 | #' @export
23 |
24 | # changelog and author contributions / copyrights
25 | # Claire Lunch (2020-09-02)
26 | # duplication of getAvg() to match new nomenclature
27 | ##############################################################################################
28 |
29 | getTimeIndex <- function(dpID, token = NA_character_) {
30 |
31 | # error message if dpID isn't formatted as expected
32 | if(regexpr("DP[1-4]{1}.[0-9]{5}.[0-9]{3}",dpID)!=1) {
33 | stop(paste(dpID, "is not a properly formatted data product ID. The correct format is DP#.#####.###", sep=" "))
34 | }
35 |
36 | # error message if dpID is the EC product
37 | if(dpID=="DP4.00200.001") {
38 | stop(paste(dpID, "is the bundled eddy covariance data product; download by averaging interval is not available.", sep=" "))
39 | }
40 |
41 | # error message if dpID isn't in table_types
42 | if(!dpID %in% table_types$productID) {
43 | stop(paste(dpID, "is not a supported data product ID. Check for typos.", sep=" "))
44 | }
45 |
46 | # error message if dpID isn't IS
47 | req <- getAPI(paste(apiURL = "https://data.neonscience.org/api/v0/products/", dpID, sep=""),
48 | token = token)
49 |
50 | if(is.null(req)) {
51 | message(paste("Could not verify data product. NEON API may be unavailable."))
52 | return(invisible())
53 | }
54 |
55 | avail <- jsonlite::fromJSON(httr::content(req, as='text', encoding='UTF-8'),
56 | simplifyDataFrame=TRUE, flatten=TRUE)
57 |
58 | if(avail$data$productScienceTeamAbbr %in% c("TOS","AOS","AOP") |
59 | dpID %in% c("DP1.20267.001","DP1.00101.001","DP1.00013.001","DP1.00038.001",
60 | "DP1.00096.001","DP1.00097.001","DP4.00133.001")) {
61 | stop(paste(dpID, "is not a streaming sensor (IS) data product; averaging interval is not relevant.", sep=" "))
62 | }
63 |
64 | # look up TMI in table_types
65 | tmi <- table_types$tableTMI[which(table_types$productID==dpID)]
66 | return(tmi)
67 |
68 | }
69 |
--------------------------------------------------------------------------------
/neonUtilities/R/getTitle.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get NEON data product title
3 | #'
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Create a title for a NEON data CSV file
9 | #'
10 | #' @keywords internal
11 |
12 | #' @param filename A NEON file name
13 | #' @return A title for the respective GeoCSV file
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # Changelog and author contributions / copyrights
19 | # 2018-01-09 (Christine Laney): created function
20 | ##############################################################################################
21 |
22 | getTitle <- function(filename) {
23 | splitName <- strsplit(x = filename, split = "\\.")
24 | if(length(splitName[[1]])==2){
25 | tableName <- splitName[[1]][1]
26 | title <- paste0("NEON ", tableName, " data from multiple sites and/or months")
27 | }
28 | if(length(splitName[[1]]) > 2){
29 | site <- splitName[[1]][3]
30 | dpID <- substr(filename, 15, 27)
31 | hor <- splitName[[1]][7]
32 | ver <- splitName[[1]][8]
33 | time <- paste(splitName[[1]][9], " minute resolution") #this will need updating for non met data products
34 | ym <- splitName[[1]][11]
35 | ym <- as.Date(x = paste0(ym, "-01"), format = "%Y-%m-%d")
36 | y <- format(ym, "%Y")
37 | m <- format(ym, "%B")
38 | uri <- paste0("https://data.neonscience.org/api/v0/products/", dpID)
39 | data_info <- jsonlite::fromJSON(txt = uri)
40 | dpName <- data_info$data$productName
41 | title <- paste("NEON", dpName, "data from", site, m, y, "at horizontal level", hor, "and vertical position", ver, sep = " " )
42 | }
43 | return(title)
44 | }
45 |
--------------------------------------------------------------------------------
/neonUtilities/R/getVariables.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get correct data types
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Support way to force R to read assign correct data types to each column based on variables file
9 | #'
10 | #' @keywords internal
11 | #' @param varFile A file that contains variable definitions
12 | #' @return A data frame with fieldName and assigned column class, along with table if present
13 |
14 | #' @references
15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
16 |
17 | # Changelog and author contributions / copyrights
18 | # Christine Laney (2017-09-28)
19 | ##############################################################################################
20 |
21 | getVariables <- function(varFile){
22 | d <- utils::read.csv(varFile, header = T, stringsAsFactors = F) # read in a variables file
23 | d$colClass <- rep("numeric", nrow(d)) # make a new colClass column defaulting to numeric
24 | d$colClass[which(d$dataType %in% c("string", "date", "dateTime", "uri"))] <- "character" # change to character if string or dateTime
25 | if("table" %in% names(d)){ # OS variables files have table, IS do not
26 | return(d[, c("table", "fieldName", "colClass")])
27 | }
28 | return(d[, c("fieldName","colClass")])
29 | }
30 |
--------------------------------------------------------------------------------
/neonUtilities/R/getVariablesEddy.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get variable names and units from SAE H5 files
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Extract variable names and units from SAE H5 files and return in user-friendly form. Used in stackEddy(), not intended for independent use.
9 | #'
10 | #' @keywords internal
11 | #' @param tabList A list of SAE data tables
12 | #' @return A table of variable names and units, aggregated from the input tables
13 |
14 | #' @references
15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
16 |
17 | # Changelog and author contributions / copyrights
18 | # Claire Lunch (2023-05-30)
19 | ##############################################################################################
20 |
21 | getVariablesEddy <- function(tabList){
22 |
23 | variables <- character(5)
24 | for(p in 1:length(tabList[[1]])) {
25 | if(!is.null(attributes(tabList[[1]][[p]])$unit)) {
26 | var.nm <- strsplit(names(tabList[[1]])[p],
27 | split="/", fixed=T)[[1]][c(3,4,length(strsplit(names(tabList[[1]])[p],
28 | split="/", fixed=T)[[1]]))]
29 | if(length(attributes(tabList[[1]][[p]])$unit)>1) {
30 | var.nm <- matrix(var.nm, ncol=3, nrow=length(attributes(tabList[[1]][[p]])$unit), byrow=T)
31 | if(length(attributes(tabList[[1]][[p]])$unit)==length(attributes(tabList[[1]][[p]])$names)) {
32 | var.nm <- cbind(var.nm, attributes(tabList[[1]][[p]])$names)
33 | } else {
34 | if("index" %in% attributes(tabList[[1]][[p]])$names) {
35 | var.nm <- cbind(var.nm,
36 | attributes(tabList[[1]][[p]])$names[-which(attributes(tabList[[1]][[p]])$names=="index")])
37 | } else {
38 | var.nm <- cbind(var.nm,
39 | attributes(tabList[[1]][[p]])$names[-which(attributes(tabList[[1]][[p]])$names
40 | %in% c("timeBgn","timeEnd"))])
41 | }
42 | }
43 | var.nm <- cbind(var.nm, attributes(tabList[[1]][[p]])$unit)
44 | variables <- rbind(variables, var.nm)
45 | } else {
46 | variables <- rbind(variables, c(var.nm, "", attributes(tabList[[1]][[p]])$unit))
47 | }
48 | }
49 | }
50 | variables <- data.frame(unique(variables))
51 |
52 | if(nrow(variables)==1) {
53 | variables <- NA
54 | } else {
55 | variables <- variables[-1,]
56 | colnames(variables) <- c("category","system","variable","stat","units")
57 | rownames(variables) <- 1:nrow(variables)
58 | }
59 |
60 | return(variables)
61 | }
62 |
63 |
--------------------------------------------------------------------------------
/neonUtilities/R/getVarsEddy.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Extract list of eddy covariance tables from HDF5 files
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Extracts a list of table metadata from a single HDF5 file. Specific to eddy covariance data product: DP4.00200.001. Can inform inputs to stackEddy(); variables listed in 'name' are available inputs to the 'var' parameter in stackEddy().
9 | #'
10 | #' @param filepath The folder containing the H5 file [character]
11 |
12 | #' @return A data frame of the metadata for each data table in the HDF5 file
13 |
14 | #' @examples
15 | #' \dontrun{
16 | #' # read variables from a file in a hypothetical filepath
17 | #' ec.vars <- getVarsEddy(filepath='/data/NEON.D19.BONA.DP4.00200.001.nsae.2017-12.basic.h5')
18 | #' }
19 |
20 | #' @references
21 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 |
23 | #' @export
24 |
25 | # changelog and author contributions / copyrights
26 | # Claire Lunch (2019-06-07)
27 | # partially adapted from eddy4R.base::def.hdf5.extr() authored by David Durden
28 | ##############################################################################################
29 |
30 | getVarsEddy <- function(filepath) {
31 |
32 | # first check for rhdf5 package
33 | if(!requireNamespace("rhdf5", quietly=T)) {
34 | stop("Package rhdf5 is required for this function to work.
35 | \nrhdf5 is a Bioconductor package. To install, use:\ninstall.packages('BiocManager')\nBiocManager::install('rhdf5')\n")
36 | }
37 |
38 | listObj <- base::try(rhdf5::h5ls(filepath), silent=T)
39 |
40 | if(inherits(listObj, "try-error")) {
41 | stop(paste(filepath, " could not be read.", sep=""))
42 | }
43 |
44 | listDataObj <- listObj[listObj$otype == "H5I_DATASET",]
45 |
46 | listObjSpl <- tidyr::separate(listDataObj, col="group",
47 | into=c(NA, "site", "level", "category", "system",
48 | "horvertmi", "subsys"),
49 | sep="/", fill="right")
50 | listObjSpl <- tidyr::separate(listObjSpl, col="horvertmi",
51 | into=c("hor", "ver", "tmi"),
52 | sep="_", fill="left")
53 | listObjSpl$oth[which(is.na(suppressWarnings(as.numeric(listObjSpl$ver))))] <-
54 | listObjSpl$ver[which(is.na(suppressWarnings(as.numeric(listObjSpl$ver))))]
55 | listObjSpl$ver[which(is.na(suppressWarnings(as.numeric(listObjSpl$ver))))] <- NA
56 |
57 | return(listObjSpl)
58 |
59 | }
60 |
--------------------------------------------------------------------------------
/neonUtilities/R/listFilesInZip.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get a data frame with the names of all files within a zipped NEON data package
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Given the top level zip file, return dataframe of all of the files within it without
9 | #' unzipping the file
10 |
11 | #' @param zippath The path to a zip file
12 | #' @return A list of filenames within the given zip file
13 | #' @keywords internal
14 | #'
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # changelog and author contributions / copyrights
19 | # Christine Laney (2017-07-02)
20 | ##############################################################################################
21 |
22 | listFilesInZip <- function(zippath){
23 | utils::unzip(zipfile = zippath, list = T)
24 | }
25 |
--------------------------------------------------------------------------------
/neonUtilities/R/listZipfiles.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Get all zip file names within a zipped NEON data package
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Given the data frame of all the files within the top level zip file,
9 | #' return an array of just the zip file names (no pdf, xml, or other files).
10 | #'
11 | #' @param zippath The path to a zip file
12 | #' @return An array of all zip files contained within the focal zip file
13 | #' @keywords internal
14 | #'
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # changelog and author contributions / copyrights
19 | # Christine Laney (2017-07-02)
20 | ##############################################################################################
21 |
22 | listZipfiles <- function(zippath){
23 | df <- utils::unzip(zipfile = zippath, list = T)
24 | ns <- df[,1]
25 | fn <- ns[which(substr(ns, nchar(ns)-3, nchar(ns)) == ".zip")]
26 | return(fn)
27 | }
28 |
--------------------------------------------------------------------------------
/neonUtilities/R/makePosColumns.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Create position (horizontal and vertical) columns
3 |
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' For instrumented meteorological data products, create position (horizontal and vertical) columns based on values
9 | #' embedded in the file names.
10 | #'
11 | #' @keywords internal
12 | #' @param d A data table
13 | #' @param datafl A data file name
14 | #' @return A data table with new columns
15 |
16 | #' @references
17 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
18 |
19 | # Changelog and author contributions / copyrights
20 | # 2017-09-28 (Christine Laney): Created original function
21 | # 2018-04-13 (Christine Laney):
22 | # * Continuous stream discharge (DP4.00130.001) is an OS product in IS format, and the HOR.VER
23 | # values given (100.100) are always the same. New HOR and VER columns are not needed in the
24 | # stacked product.
25 | # 2019-10-17 (Nathan Mietkiewicz): Add domainID, siteID, and collection YYYY-MM columns for sensor position files
26 | ##############################################################################################
27 |
28 | makePosColumns <- function(d, datafl, site){
29 |
30 | datafl.splitFile <- strsplit(x = datafl, split = "\\/")
31 | datafl.splitName <- strsplit(x = datafl.splitFile[[1]][length(datafl.splitFile[[1]])], split = "\\.")
32 |
33 | sensor_positions <- grepl('sensor_positions', datafl.splitName)
34 |
35 | if((datafl.splitName[[1]][4]=="DP4") && (datafl.splitName[[1]][5]=="00130") &
36 | FALSE %in% sensor_positions){return(d)}
37 |
38 | nc <- ncol(d)
39 | if(length(datafl.splitName[[1]]) %in% c(12,14) || (TRUE %in% sensor_positions)){
40 | if(length(datafl.splitName[[1]]) == 12){
41 | horPos <- 8
42 | verPos <- 9
43 | }
44 | if(length(datafl.splitName[[1]]) == 14){
45 | horPos <- 7
46 | verPos <- 8
47 | }
48 | if(TRUE %in% sensor_positions) {
49 | d$siteID <- rep(site, nrow(d))
50 | d <- data.table::setcolorder(d, c(ncol(d), 1:I(ncol(d)-1)))
51 | } else {
52 | if(!("siteID" %in% names(d))){
53 | d$domainID <- as.character(unlist(datafl.splitName)[2])
54 | d$siteID <- as.character(unlist(datafl.splitName)[3])
55 | }
56 | d$horizontalPosition <- as.character(rep(as.character(datafl.splitName[[1]][horPos]), nrow(d)))
57 | d$verticalPosition <- as.character(rep(as.character(datafl.splitName[[1]][verPos]), nrow(d)))
58 | d <- data.table::setcolorder(d, c((nc+1):(nc+4),1:nc))
59 | }
60 | }
61 | d$publicationDate <- as.character(rep(datafl.splitName[[1]][length(datafl.splitName[[1]])-1]), nrow(d))
62 | return(d)
63 | }
--------------------------------------------------------------------------------
/neonUtilities/R/other_bundles.R:
--------------------------------------------------------------------------------
1 | #' Bundled vegetation and sediment data product information
2 | #'
3 | #' A dataset containing NEON data product codes of vegetation and sediment data products and the "home" data products they are bundled with.
4 | #'
5 | #' @format A data frame with 2 variables:
6 | #' \describe{
7 | #' \item{product}{Data product ID of a product}
8 | #' \item{homeProduct}{Data product ID of the corresponding home data product}
9 | #' }
10 | #' @source NEON data product bundles
11 | "other_bundles"
12 |
--------------------------------------------------------------------------------
/neonUtilities/R/quietMessages.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Will suppress all output messages, while retaining the output dataframe
3 |
4 | #' @author
5 | #' Nate Mietkiewicz \email{mietkiewicz@battelleecology.org}
6 |
7 | #' @description Used to quiet all output messages
8 | #'
9 | #' @param toBeQuieted Input to be quieted
10 |
11 | #' @return The expected output without associated messages/warnings.
12 | #' @keywords internal
13 |
14 | #' @references
15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
16 |
17 | # Changelog and author contributions / copyrights
18 | # Nate Mietkiewcz (2020-02-27): original creation
19 |
20 | ##############################################################################################
21 |
22 | quietMessages <- function(toBeQuieted) {
23 | sink(tempfile())
24 | on.exit(sink())
25 | invisible(force(toBeQuieted))
26 | }
27 |
--------------------------------------------------------------------------------
/neonUtilities/R/readTableNEON.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Read a NEON data table with correct data types for each variable
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Load a table into R, assigning classes to each column based on data types in variables file; or convert a table already loaded
9 | #'
10 | #' @param dataFile A data frame containing a NEON data table, or the filepath to a data table to load
11 | #' @param varFile A data frame containing the corresponding NEON variables file, or the filepath to the variables file
12 | #' @param useFasttime Should the fasttime package be used to read date-time variables? Defaults to false.
13 | #' @return A data frame of a NEON data table, with column classes assigned by data type
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 | #'
18 | #' @export
19 |
20 | # Changelog and author contributions / copyrights
21 | # Claire Lunch (2019-11-08)
22 | ##############################################################################################
23 |
24 | readTableNEON <- function(dataFile, varFile, useFasttime=FALSE){
25 |
26 | # check for fasttime package, if used
27 | if(useFasttime & !requireNamespace("fasttime", quietly=T)) {
28 | stop("Parameter useFasttime is TRUE but fasttime package is not installed. Install and re-try.")
29 | }
30 |
31 | # read in variables file
32 | if(inherits(varFile, 'character')) {
33 | v <- utils::read.csv(varFile, header = T, stringsAsFactors = F,
34 | na.strings=c('NA',''), encoding='UTF-8')
35 | } else {
36 | v <- try(as.data.frame(varFile), silent=T)
37 | if(inherits(v, 'try-error')) {
38 | stop('varFile must be either a NEON variables table or a file path to a NEON variables table.\n')
39 | }
40 | }
41 |
42 | # check this is a valid variables file
43 | if(any(c('category','system','stat') %in% names(v))) {
44 | stop('varFile appears to match DP4.00200.001. Use the stackEddy() function to work with surface-atmosphere exchange data.')
45 | } else {
46 | if(any(!c('table','fieldName','dataType') %in% names(v))) {
47 | stop('varFile is not a variables file, or is missing critical values.\n')
48 | }
49 | }
50 |
51 | # make a new colClass column defaulting to numeric
52 | # modify to character for strings and urls
53 | v$colClass <- rep("numeric", nrow(v))
54 | v$colClass[which(v$dataType %in% c("string","uri"))] <- "character"
55 | v$colClass[which(v$dataType %in% c("date","dateTime"))] <- 'date'
56 |
57 | v <- v[, c("table", "fieldName", "colClass")]
58 |
59 | # read in data file
60 | if(inherits(dataFile, 'character')) {
61 | d <- suppressWarnings(utils::read.csv(dataFile, header=T, stringsAsFactors=F,
62 | colClasses=c(horizontalPosition='character', verticalPosition='character'),
63 | na.strings=c('NA',''), encoding='UTF-8'))
64 | } else {
65 | d <- try(as.data.frame(dataFile, stringsAsFactors=F), silent=T)
66 | if(inherits(d, 'try-error')) {
67 | stop('dataFile must be either a NEON data table or a file path to a NEON data table.\n')
68 | }
69 | }
70 |
71 | # check that most fields have a corresponding value in variables
72 | m <- length(which(!names(d) %in% v$fieldName))
73 | if(m==length(names(d))) {
74 | stop('Variables file does not match data file.\n')
75 | }
76 | if(m>4) {
77 | message(paste(m, " fieldNames are present in data files but not in variables file. Unknown fields are read as character strings.\n", sep=""))
78 | }
79 |
80 | # fieldNames each have a unique dataType - don't need to match table
81 | for(i in names(d)) {
82 | if(!i %in% v$fieldName) {
83 | d[,i] <- as.character(d[,i])
84 | } else {
85 | type <- v$colClass[which(v$fieldName==i)][1]
86 | if(type=='numeric') {
87 | d[,i] <- as.numeric(d[,i])
88 | }
89 | if(type=='character') {
90 | d[,i] <- as.character(d[,i])
91 | }
92 | if(type=='date') {
93 | d[,i] <- dateConvert(d[,i], useFasttime=useFasttime)
94 | }
95 | }
96 | }
97 |
98 | return(d)
99 | }
100 |
--------------------------------------------------------------------------------
/neonUtilities/R/schemaAllStrings.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Create an arrow schema with every variable coded as a string field.
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Use the field names in a NEON variables file to create an arrow schema of all strings, or, if no variables file is available, read the header and assign everything as string.
9 | #'
10 | #' @param variables A data frame containing a NEON variables file for a single table, or a set of field names.
11 | #' @keywords internal
12 | #'
13 | #' @return An arrow schema for the relevant data table with all variables set to string.
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | # Changelog and author contributions / copyrights
19 | # Claire Lunch (2025-03-12)
20 | ##############################################################################################
21 |
22 | schemaAllStrings <- function(variables) {
23 |
24 | # still need to build in option for set of names, without full variables file
25 | # schemaFromVar() is now set to only call this function if the variables file is available
26 | # so only need the names option if this function is called from elsewhere in the future
27 | stringschema <- arrow::schema(unlist(apply(variables, MARGIN=1, FUN=function(x) {
28 | arrow::field(name=x["fieldName"], type=arrow::string())
29 | })))
30 |
31 | return(stringschema)
32 |
33 | }
34 |
35 |
--------------------------------------------------------------------------------
/neonUtilities/R/schemaFromVar.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Create an arrow schema from a NEON variables file.
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Use the field names and data types in a NEON variables file to create an arrow schema.
9 | #'
10 | #' @param variables A data frame containing a NEON variables file, or a url pointing to a NEON variables file.
11 | #' @param tab The name of the table to generate a schema from.
12 | #' @param package Should the schema be created for the basic or expanded package?
13 | #'
14 | #' @return An arrow schema for the relevant data table.
15 | #'
16 | #' @export
17 |
18 | #' @references
19 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 |
21 | # Changelog and author contributions / copyrights
22 | # Claire Lunch (2025-03-11)
23 | ##############################################################################################
24 |
25 | schemaFromVar <- function(variables, tab, package) {
26 |
27 | unify <- FALSE
28 |
29 | # is the input a file or a url?
30 | if(inherits(variables, "character")) {
31 | # read in variables file
32 | vartab <- try(data.frame(arrow::read_csv_arrow(variables)), silent=TRUE)
33 | vartab <- try(vartab[which(vartab$table==tab),], silent=TRUE)
34 | if(inherits(vartab, "try-error")) {
35 | message("There was a problem reading the variables file. Data types will be inferred.")
36 | unify <- TRUE
37 | } else {
38 | if(nrow(vartab)==0) {
39 | message("There was a problem reading the variables file. Data types will be inferred.")
40 | unify <- TRUE
41 | }
42 | }
43 | } else {
44 | vartab <- try(variables[which(variables$table==tab),], silent=TRUE)
45 | if(inherits(vartab, "try-error")) {
46 | message("There was a problem reading the variables file. Data types will be inferred.")
47 | unify <- TRUE
48 | }
49 | }
50 |
51 | # if reading the file failed, function returns NULL
52 | if(unify) {
53 | vschema <- NULL
54 | } else {
55 |
56 | # if working with the basic package, subset the table
57 | if(package=="basic") {
58 | vartab <- vartab[which(vartab$downloadPkg=="basic"),]
59 | }
60 |
61 | # start by making a schema with everything as a string
62 | vschema <- schemaAllStrings(vartab)
63 |
64 | # translate data types to arrow types for non-character fields
65 | ind <- which(!vartab$dataType %in% c("string", "uri"))
66 | for(i in ind) {
67 | if(vartab$dataType[i]=="real") {
68 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i], type=arrow::float64())
69 | }
70 | if(vartab$dataType[i] %in% c("integer", "unsigned integer",
71 | "signed integer")) {
72 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i], type=arrow::int64())
73 | }
74 | if(vartab$dataType[i]=="dateTime" &
75 | vartab$pubFormat[i] %in% c("yyyy-MM-dd'T'HH:mm:ss'Z'(floor)",
76 | "yyyy-MM-dd'T'HH:mm:ss'Z'",
77 | "yyyy-MM-dd'T'HH:mm:ss'Z'(round)")) {
78 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i],
79 | type=arrow::timestamp("s", timezone="UTC"))
80 | }
81 | if(vartab$dataType[i]=="dateTime" &
82 | vartab$pubFormat[i] %in% c("yyyy-MM-dd(floor)", "yyyy-MM-dd")) {
83 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i], type=arrow::date32())
84 | }
85 | if(vartab$dataType[i]=="dateTime" &
86 | vartab$pubFormat[i] %in% c("yyyy(floor)", "yyyy(round)")) {
87 | vschema[[i]] <- arrow::field(name=vartab$fieldName[i], type=arrow::int64())
88 | }
89 |
90 | }
91 |
92 | }
93 |
94 | return(vschema)
95 |
96 | }
97 |
98 |
--------------------------------------------------------------------------------
/neonUtilities/R/shared_aquatic.R:
--------------------------------------------------------------------------------
1 | #' Terrestrial-aquatic shared data information
2 | #'
3 | #' A dataset containing NEON site codes and data product IDs for places where meteorological data from terrestrial sites are used as the data of record for nearby aquatic sites as well.
4 | #'
5 | #' @format A data frame with 3 variables:
6 | #' \describe{
7 | #' \item{site}{site code of a NEON aquatic site}
8 | #' \item{towerSite}{site code of the NEON terrestrial site used as the data source for the corresponding aquatic site}
9 | #' \item{product}{Data product ID of the data products to which the corresponding terrestrial-aquatic site relationship relates}
10 | #' }
11 | #' @source NEON site layouts and spatial design
12 | "shared_aquatic"
13 |
--------------------------------------------------------------------------------
/neonUtilities/R/shared_flights.R:
--------------------------------------------------------------------------------
1 | #' Flight coverage information
2 | #'
3 | #' A dataset containing NEON site codes for places where a single AOP flight may cover multiple sites
4 | #'
5 | #' @format A data frame with 2 variables:
6 | #' \describe{
7 | #' \item{site}{site code of a NEON site}
8 | #' \item{flightSite}{site code that matches the file naming for flights that may include "site"}
9 | #' }
10 | #' @source NEON flight plans
11 | "shared_flights"
12 |
--------------------------------------------------------------------------------
/neonUtilities/R/sysdata.rda:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/R/sysdata.rda
--------------------------------------------------------------------------------
/neonUtilities/R/table_types.R:
--------------------------------------------------------------------------------
1 | #' Publication table information
2 | #'
3 | #' A dataset containing publication table names, descriptions, type (site-date, site-all, lab-all, lab-current), and a time index
4 | #'
5 | #' @format A data frame with 5 variables. Number of rows changes frequently as more tables are added:
6 | #' \describe{
7 | #' \item{productID}{data product ID}
8 | #' \item{tableName}{name of table}
9 | #' \item{tableDesc}{description of table}
10 | #' \item{tableType}{type of table (important for knowing which tables to stack, and how to stack)}
11 | #' \item{tableTMI}{a time index (e.g., 0 = native resolution, 1 = 1 minute, 30 = 30 minute averages or totals)}
12 | #' }
13 | #' @source NEON database
14 | "table_types"
15 |
--------------------------------------------------------------------------------
/neonUtilities/R/timeStampSet.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Generate a consensus set of time stamps from a set of input tables.
3 |
4 | #' @author
5 | #' Claire Lunch \email{clunch@battelleecology.org}
6 |
7 | #' @description
8 | #' Generate consensus SAE time stamps from a set of tables. Used in stackEddy(), not intended for independent use.
9 | #'
10 | #' @keywords internal
11 | #' @param tabList A list of SAE data tables
12 | #' @return A table of time stamps (start and end times) aggregated from the input tables
13 |
14 | #' @references
15 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
16 |
17 | # Changelog and author contributions / copyrights
18 | # Claire Lunch (2023-05-30)
19 | ##############################################################################################
20 |
21 | timeStampSet <- function(tabList){
22 |
23 | nameSet <- c("timeBgn","timeEnd")
24 |
25 | # get a set of time stamps to initiate the table. leave out qfqm to exclude
26 | # filler records created as placeholders for days with no data
27 | timeSet <- tabList[grep("qfqm", names(tabList), invert=T)]
28 | # turbulent flux and footprint end time stamps don't quite match the others
29 | timeSet <- timeSet[grep("turb", names(timeSet), invert=T)]
30 | timeSet <- timeSet[grep("foot", names(timeSet), invert=T)]
31 |
32 | # initiate the table with consensus set of time stamps
33 | timeSetInit <- timeSet[[1]][,nameSet]
34 | if(length(timeSet)==1) {
35 | timeSetInit <- timeSetInit
36 | } else {
37 | for(q in 2:length(timeSet)) {
38 | # check for additional start time stamps
39 | timeSetTemp <- timeSet[[q]][,nameSet]
40 | timeSetTempMerg <- data.table::as.data.table(timeSetTemp[,"timeBgn"])
41 | timeSetInitMerg <- data.table::as.data.table(timeSetInit[,"timeBgn"])
42 | misTime <- data.table::fsetdiff(timeSetTempMerg, timeSetInitMerg)
43 | if(nrow(misTime)==0) {
44 | timeSetInit <- timeSetInit
45 | } else {
46 | # combine all, then de-dup
47 | allTime <- data.table::rbindlist(list(timeSetInit, timeSetTemp), fill=TRUE)
48 | timeSetInit <- as.data.frame(unique(allTime, by="timeBgn"))
49 | }
50 | }
51 | }
52 |
53 | return(data.table::as.data.table(timeSetInit))
54 |
55 | }
56 |
57 |
--------------------------------------------------------------------------------
/neonUtilities/R/transformFileToGeoCSV.R:
--------------------------------------------------------------------------------
1 | ##############################################################################################
2 | #' @title Transform NEON CSV file to GeoCSV
3 | #'
4 | #' @author
5 | #' Christine Laney \email{claney@battelleecology.org}
6 |
7 | #' @description
8 | #' Read in a NEON monthly data zip file and parse the respective variables file to create a new GeoCSV file
9 | #'
10 | #' @param infile The path to the file that needs to be parsed
11 | #' @param varfile The path to the variables file needed to parse the infile
12 | #' @param outfile The path where the new GeoCSV file should be placed
13 | #' @return The same data file with a GeoCSV header
14 |
15 | #' @references
16 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 |
18 | #' @export
19 |
20 | # Changelog and author contributions / copyrights
21 | # 2018-01-09 (Christine Laney): Created function
22 | # 2018-04-04 (Christine Laney): Updated function documentation
23 | # 2018-04-17 (Christine Laney): Added field descriptions under the keyword field_long_name
24 | ##############################################################################################
25 |
26 | transformFileToGeoCSV <- function(infile, varfile, outfile){
27 | datafile <- utils::read.csv(infile, header = TRUE, stringsAsFactors = FALSE)
28 | vars <- utils::read.csv(varfile, header = TRUE, stringsAsFactors = FALSE)
29 | dfsplit <- strsplit(infile, "\\/")
30 | dfname <- dfsplit[[1]][length(dfsplit[[1]])]
31 |
32 | line_dataset <- "# dataset: GeoCSV 2.0\n"
33 | line_title <- paste0("# title: ", getTitle(dfname))
34 | line_institution <- "# institution: National Ecological Observatory Network (NEON)"
35 | line_attribution <- "# attribution: http://www.neonscience.org/data-resources/data-usage-citations"
36 | line_resource <- "# resource: http://data.neonscience.org"
37 | line_field_long_name <- "# field_long_name: "
38 | line_field_unit <- "# field_unit: "
39 | line_field_type <- "# field_type: "
40 | line_delimiter <- '# delimiter: ","'
41 | for(i in 1:ncol(datafile)){
42 | type <- vars$dataType[which(names(datafile)[i] == vars$fieldName)][1]
43 | if(is.na(type)){type = "NA"}
44 | if(type == "dateTime"){unit = "ISO_8601"}
45 | if(type %in% c("unsigned integer","signed integer")){type = "integer"}
46 | if(type == "real"){type = "float"}
47 | if(!(type %in% c("ISO_8601","integer","float"))){type = "string"}
48 |
49 | unit <- vars$units[which(names(datafile)[i] == vars$fieldName)][1]
50 | if(is.na(unit) || unit == "NA"){unit = "unitless"}
51 |
52 | long_name <- vars$description[which(names(datafile)[i] == vars$fieldName)][1]
53 |
54 | if(i == 1){
55 | line_field_long_name <- paste0(line_field_long_name, long_name)
56 | line_field_unit <- paste0(line_field_unit, unit)
57 | line_field_type <- paste0(line_field_type, type)
58 | }
59 | if(i > 1){
60 | line_field_long_name <- paste(line_field_long_name, long_name, sep = ", ")
61 | line_field_type <- paste(line_field_type, type, sep = ", ")
62 | line_field_unit <- paste(line_field_unit, unit, sep = ", ")
63 | }
64 | }
65 |
66 | addText <- function(object, outfile, sep = "\n", append = TRUE){
67 | cat(object, file = outfile, sep = sep, append = append)
68 | }
69 | addText(line_dataset, outfile, append = FALSE)
70 | addText(line_title, outfile)
71 | addText(line_institution, outfile)
72 | addText(line_attribution, outfile)
73 | addText(line_resource, outfile)
74 | addText(line_field_long_name, outfile)
75 | addText(line_field_type, outfile)
76 | addText(line_field_unit, outfile)
77 | utils::write.table(datafile, file = outfile, row.names = FALSE, append = TRUE, sep = ",",
78 | fileEncoding = "UTF-8", quote = FALSE)
79 | }
80 |
81 |
82 |
--------------------------------------------------------------------------------
/neonUtilities/data-raw/update_table_types.R:
--------------------------------------------------------------------------------
1 | # update_table_types.R
2 | ##############################################################################################
3 | #' @title Update table_types data frame in sysdata.rda
4 |
5 | #' @author
6 | #' Christine Laney \email{claney@battelleecology.org}
7 |
8 | #' @description
9 | #' Connect to the DPS database, pull in updated information about tables, and save locally.
10 |
11 | #' @return A saved .rda file
12 |
13 | #' @references
14 | #' License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
15 |
16 | # Changelog and author contributions / copyrights
17 | # Claire Lunch (2018-10-05) converted to get tables from PDR via the data service
18 | # Christine Laney (2017-10-19)
19 | ##############################################################################################
20 |
21 | update_table_types <- function(){
22 |
23 | options(stringsAsFactors=F)
24 |
25 | # get publication tables from PDR
26 | req <- httr::GET(Sys.getenv("PUB_TABLES"))
27 | rawx <- XML::xmlToList(httr::content(req, as="text"))
28 |
29 | ids <- substring(unlist(lapply(rawx, '[', "dataProductId")), 15, 27)
30 | tables <- unlist(lapply(rawx, '[', "tableName"))
31 | tables <- gsub("_pub", "", tables, fixed=T)
32 | descs <- unlist(lapply(rawx, '[', "description"))
33 | typs <- unlist(lapply(rawx, '[', "tableType"))
34 | temp <- unlist(lapply(rawx, '[', "pubField"))
35 | tmi <- temp[grep("timeIndex", names(temp))]
36 |
37 | table_types <- data.frame(cbind(ids[-length(ids)], tables[-length(tables)],
38 | descs[-length(descs)], typs[-length(typs)], tmi))
39 | colnames(table_types) <- c("productID", "tableName", "tableDesc",
40 | "tableType", "tableTMI")
41 |
42 | # need to keep table definitions for tables that only exist in RELEASE-2021
43 | table_types <- rbind(table_types, release_2021)
44 |
45 | rownames(table_types) <- 1:nrow(table_types)
46 |
47 | # term definitions for fields added by stackByTable
48 | added_fields <- data.frame(cbind(fieldName=c('domainID','siteID','horizontalPosition',
49 | 'verticalPosition','publicationDate',
50 | 'release'),
51 | description=c('Unique identifier of the NEON domain',
52 | 'NEON site code',
53 | 'Index of horizontal location at a NEON site',
54 | 'Index of vertical location at a NEON site',
55 | 'Date of data publication on the NEON data portal',
56 | 'Identifier for data release'),
57 | dataType=c(rep('string',4),'dateTime','string'),
58 | units=rep(NA,6),
59 | downloadPkg=rep('appended by stackByTable',6),
60 | pubFormat=rep(NA,6),
61 | primaryKey=rep('N',6),
62 | categoricalCodeName=rep('',6)))
63 |
64 | usethis::use_data(table_types, added_fields, science_review_variables, shared_flights,
65 | shared_aquatic, release_2021, chem_bundles, other_bundles, relevant_EPSG,
66 | cit_prov_template, internal=TRUE, overwrite=TRUE)
67 | usethis::use_data(table_types, shared_flights, shared_aquatic, chem_bundles, other_bundles,
68 | internal=FALSE, overwrite=TRUE)
69 |
70 | # write tables to repo for use outside package
71 | data.table::fwrite(added_fields, paste(Sys.getenv('NU_REPO'), 'helper_files/added_fields.csv', sep='/'))
72 | data.table::fwrite(chem_bundles, paste(Sys.getenv('NU_REPO'), 'helper_files/chem_bundles.csv', sep='/'))
73 | data.table::fwrite(other_bundles, paste(Sys.getenv('NU_REPO'), 'helper_files/other_bundles.csv', sep='/'))
74 | data.table::fwrite(release_2021, paste(Sys.getenv('NU_REPO'), 'helper_files/release_2021.csv', sep='/'))
75 | data.table::fwrite(shared_aquatic, paste(Sys.getenv('NU_REPO'), 'helper_files/shared_aquatic.csv', sep='/'))
76 | data.table::fwrite(shared_flights, paste(Sys.getenv('NU_REPO'), 'helper_files/shared_flights.csv', sep='/'))
77 | data.table::fwrite(table_types, paste(Sys.getenv('NU_REPO'), 'helper_files/table_types.csv', sep='/'))
78 |
79 | }
80 |
81 | update_table_types()
82 |
--------------------------------------------------------------------------------
/neonUtilities/data/chem_bundles.rda:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/chem_bundles.rda
--------------------------------------------------------------------------------
/neonUtilities/data/other_bundles.rda:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/other_bundles.rda
--------------------------------------------------------------------------------
/neonUtilities/data/shared_aquatic.rda:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/shared_aquatic.rda
--------------------------------------------------------------------------------
/neonUtilities/data/shared_flights.rda:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/shared_flights.rda
--------------------------------------------------------------------------------
/neonUtilities/data/table_types.rda:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/data/table_types.rda
--------------------------------------------------------------------------------
/neonUtilities/inst/extdata/NEON_gp.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/neonUtilities/inst/extdata/NEON_gp.zip
--------------------------------------------------------------------------------
/neonUtilities/inst/extdata/expected/gp/issueLog_10109.csv:
--------------------------------------------------------------------------------
1 | "id","parentIssueID","issueDate","resolvedDate","dateRangeStart","dateRangeEnd","locationAffected","issue","resolution"
2 | 16095,NA,"2020-06-01T00:00:00Z","2020-06-01T00:00:00Z","2018-11-01T00:00:00Z","2020-06-01T00:00:00Z","All terrestrial sites","After consultation with external working group community members and the NSF, The Soil Microbe Group Abundances data product was discontinued in 2020, with the last field samples having associated group abundance data in 10-2018. Users who are interested in generating this data from NEON samples may submit a request for samples or DNA extracts from the NEON Biorepository. Alternatively, the Soil Microbe Biomass data product (DP1.10104) may be utilized as a similar measurement of the quantities of microbes in soils.","Data Product discontinued"
3 |
--------------------------------------------------------------------------------
/neonUtilities/inst/extdata/expected/gp/mga_labSummary.csv:
--------------------------------------------------------------------------------
1 | uid,targetTaxonGroup,labSpecificStartDate,labSpecificEndDate,laboratoryName,qpcrDetectionMethod,primerConcentration,forwardPrimer,reversePrimer,ampliconLength,dNTPConcentration,probeConcentration,mgConcentration,polymeraseType,polymeraseUnitNumber,bufferID,bufferComposition,reactionVolume,qpcrMethod,instrument,primerSpecificity,detectionLimit,multiplexStatus,qPCRAnalysisProgram,cqMethod,recordedBy,dataQF,publicationDate
2 | 91969727-5313-4707-b6d4-2aaeecaa01ab,fungi,2017-12-08,2018-02-25,Battelle Applied Genomics,DNA binding dye,0.2,CTTGGTCATTTAGAGGAAGTAA,GCTGCGTTCTTCATCGATGC,325,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v3,ABI 7500 Fast,,4,N,ABI 7500 v2.3,BMI_qPCR_SOP_v3,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z
3 | fccbb845-2684-486c-b44f-a47fd27713dc,bacteria and archaea,2017-12-08,2018-02-25,Battelle Applied Genomics,DNA binding dye,0.2,CCTACGGGNBGCASCAG,GGACTACNVGGGTATCTAATCC,467,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v3,ABI 7500 Fast,,22,N,ABI 7500 v2.3,BMI_qPCR_SOP_v3,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z
4 | 49cb98f9-8277-4d39-87bc-316ff97637b8,bacteria and archaea,2017-12-08,2018-02-25,Battelle Applied Genomics,DNA binding dye,0.2,CCTACGGGNBGCASCAG,GGACTACNVGGGTATCTAATCC,467,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v3,ABI 7500 Fast,,22,N,ABI 7500 v2.3,BMI_qPCR_SOP_v2,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z
5 | 3f96ee99-887b-48d9-bcc6-0485cf67e24a,fungi,2017-12-08,2018-02-25,Battelle Applied Genomics,DNA binding dye,0.2,CTTGGTCATTTAGAGGAAGTAA,GCTGCGTTCTTCATCGATGC,325,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v3,ABI 7500 Fast,,4,N,ABI 7500 v2.3,BMI_qPCR_SOP_v2,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z
6 | ff9c7fdb-06db-429a-bcb8-58f9e3f63122,bacteria and archaea,2017-09-01,2017-11-14,Battelle Applied Genomics,DNA binding dye,0.2,CCTACGGGNBGCASCAG,GGACTACNVGGGTATCTAATCC,467,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v1,ABI 7500 Fast,,22,N,ABI 7500 v2.3,BMI_qPCR_SOP_v1,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z
7 | 3fb531f3-f932-449f-b8da-6e239c9828bc,fungi,2017-09-01,2017-11-14,Battelle Applied Genomics,DNA binding dye,0.2,CTTGGTCATTTAGAGGAAGTAA,GCTGCGTTCTTCATCGATGC,325,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v1,ABI 7500 Fast,,4,N,ABI 7500 v2.3,BMI_qPCR_SOP_v1,rxnG2CiGqmvDdHh9VY28xQ==,,20180705T173721Z
8 | 67adffbb-3049-425b-aee0-6cae37e7e9d8,bacteria and archaea,2017-11-15,2017-12-08,Battelle Applied Genomics,DNA binding dye,0.2,CCTACGGGNBGCASCAG,GGACTACNVGGGTATCTAATCC,467,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v2,ABI 7500 Fast,,22,N,ABI 7500 v2.3,BMI_qPCR_SOP_v2,TuIbGgXF7vfkyq5thAKFMA==,,20180705T173721Z
9 | 9c91ab5b-ff3a-44fa-86e2-72e29c5748b3,fungi,2017-11-15,2017-12-08,Battelle Applied Genomics,DNA binding dye,0.2,CTTGGTCATTTAGAGGAAGTAA,GCTGCGTTCTTCATCGATGC,325,1.2,,2.5,KAPA,,,,20,BMI_qPCR_SOP_v2,ABI 7500 Fast,,4,N,ABI 7500 v2.3,BMI_qPCR_SOP_v2,TuIbGgXF7vfkyq5thAKFMA==,,20180705T173721Z
10 |
--------------------------------------------------------------------------------
/neonUtilities/man/assignClasses.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/assignClasses.R
3 | \name{assignClasses}
4 | \alias{assignClasses}
5 | \title{Assign correct column classes}
6 | \usage{
7 | assignClasses(dt, inVars)
8 | }
9 | \arguments{
10 | \item{dt}{A data frame}
11 |
12 | \item{inVars}{The variables expected in the df}
13 | }
14 | \value{
15 | A data frame with corrected column classes
16 | }
17 | \description{
18 | Use the variables file to assign classes to each column in each data file
19 | }
20 | \references{
21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 | }
23 | \author{
24 | Christine Laney \email{claney@battelleecology.org}
25 | }
26 | \keyword{internal}
27 |
--------------------------------------------------------------------------------
/neonUtilities/man/byEventSIM.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/byEventSIM.R
3 | \name{byEventSIM}
4 | \alias{byEventSIM}
5 | \title{Get site management data by event type.}
6 | \usage{
7 | byEventSIM(
8 | eventType,
9 | site = "all",
10 | startdate = NA,
11 | enddate = NA,
12 | release = "current",
13 | include.provisional = FALSE,
14 | token = NA_character_
15 | )
16 | }
17 | \arguments{
18 | \item{eventType}{The value of eventType to search for. Can be multiple values. See categoricalCodes file for DP1.10111.001 for possible values.}
19 |
20 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.}
21 |
22 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
23 |
24 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
25 |
26 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.}
27 |
28 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.}
29 |
30 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
31 | }
32 | \value{
33 | A data frame of sim_eventData data, matching the query criteria. Note that metadata are not included in the data returned via this function.
34 | }
35 | \description{
36 | Query site management data to return records matching a specific eventType.
37 | }
38 | \examples{
39 | \dontrun{
40 | # Search for fires across all NEON event data
41 | sim.fires <- byEventSIM(eventType="fire")
42 |
43 | # Search for grazing events at several sites
44 | sim.graz <- byEventSIM(eventType="grazing", site=c("CPER","KONA","MOAB","STER","LAJA"))
45 | }
46 |
47 | }
48 | \references{
49 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
50 | }
51 | \author{
52 | Claire Lunch \email{clunch@battelleecology.org}
53 | }
54 |
--------------------------------------------------------------------------------
/neonUtilities/man/byFileAOP.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/byFileAOP.R
3 | \name{byFileAOP}
4 | \alias{byFileAOP}
5 | \title{Serially download all AOP files for a given site, year, and product}
6 | \usage{
7 | byFileAOP(
8 | dpID,
9 | site,
10 | year,
11 | include.provisional = FALSE,
12 | check.size = TRUE,
13 | savepath = NA,
14 | token = NA_character_
15 | )
16 | }
17 | \arguments{
18 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001}
19 |
20 | \item{site}{The four-letter code of a single NEON site, e.g. 'CLBJ'.}
21 |
22 | \item{year}{The four-digit year to search for data. Defaults to 2017.}
23 |
24 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.}
25 |
26 | \item{check.size}{T or F, should the user approve the total file size before downloading? Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.}
27 |
28 | \item{savepath}{The file path to download to. Defaults to NA, in which case the working directory is used.}
29 |
30 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
31 | }
32 | \value{
33 | A folder in the working directory, containing all files meeting query criteria.
34 | }
35 | \description{
36 | Query the API for AOP data by site, year, and product, and download all files found, preserving original
37 | folder structure. Downloads serially to avoid overload; may take a very long time.
38 | }
39 | \examples{
40 | \dontrun{
41 | # To download 2017 vegetation index data from San Joaquin Experimental Range:
42 | byFileAOP(dpID="DP3.30026.001", site="SJER", year="2017")
43 | }
44 | }
45 | \references{
46 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
47 | }
48 | \author{
49 | Claire Lunch \email{clunch@battelleecology.org}
50 | Christine Laney \email{claney@battelleecology.org}
51 | }
52 |
--------------------------------------------------------------------------------
/neonUtilities/man/byTileAOP.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/byTileAOP.R
3 | \name{byTileAOP}
4 | \alias{byTileAOP}
5 | \title{Download AOP tiles overlapping specified coordinates for a given site, year, and product}
6 | \usage{
7 | byTileAOP(
8 | dpID,
9 | site,
10 | year,
11 | easting,
12 | northing,
13 | buffer = 0,
14 | include.provisional = FALSE,
15 | check.size = TRUE,
16 | savepath = NA,
17 | token = NA_character_
18 | )
19 | }
20 | \arguments{
21 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001}
22 |
23 | \item{site}{The four-letter code of a single NEON site, e.g. 'CLBJ'.}
24 |
25 | \item{year}{The four-digit year to search for data. Defaults to 2017.}
26 |
27 | \item{easting}{A vector containing the easting UTM coordinates of the locations to download.}
28 |
29 | \item{northing}{A vector containing the northing UTM coordinates of the locations to download.}
30 |
31 | \item{buffer}{Size, in meters, of the buffer to be included around the coordinates when determining which tiles to download. Defaults to 0. If easting and northing coordinates are the centroids of NEON TOS plots, use buffer=20.}
32 |
33 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.}
34 |
35 | \item{check.size}{T or F, should the user approve the total file size before downloading? Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.}
36 |
37 | \item{savepath}{The file path to download to. Defaults to NA, in which case the working directory is used.}
38 |
39 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
40 | }
41 | \value{
42 | A folder in the working directory, containing all files meeting query criteria.
43 | }
44 | \description{
45 | Query the API for AOP data by site, year, product, and tile location, and download all files found.
46 | Downloads serially to avoid overload; may take a very long time.
47 | }
48 | \references{
49 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
50 | }
51 | \author{
52 | Claire Lunch \email{clunch@battelleecology.org}
53 | Christine Laney \email{claney@battelleecology.org}
54 | }
55 |
--------------------------------------------------------------------------------
/neonUtilities/man/chem_bundles.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/chem_bundles.R
3 | \docType{data}
4 | \name{chem_bundles}
5 | \alias{chem_bundles}
6 | \title{Bundled chemistry data product information}
7 | \format{
8 | A data frame with 2 variables:
9 | \describe{
10 | \item{product}{Data product ID of a terrestrial chemistry product}
11 | \item{homeProduct}{Data product ID of the corresponding home data product}
12 | }
13 | }
14 | \source{
15 | NEON data product bundles
16 | }
17 | \usage{
18 | chem_bundles
19 | }
20 | \description{
21 | A dataset containing NEON data product codes of terrestrial chemistry data products and the "home" data products they are bundled with.
22 | }
23 | \keyword{datasets}
24 |
--------------------------------------------------------------------------------
/neonUtilities/man/cleanUp.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/cleanUp.R
3 | \name{cleanUp}
4 | \alias{cleanUp}
5 | \title{Clean up folder after stacking}
6 | \usage{
7 | cleanUp(folder, orig)
8 | }
9 | \arguments{
10 | \item{folder}{The file path to the folder that needs to be cleaned up (the root directory of the data package)}
11 |
12 | \item{orig}{The list of files that were present in the folder before unzipping and stacking}
13 | }
14 | \value{
15 | Only the folders created during unzip will be deleted. All custom folders/files and the stackedFiles output folder will be retained.
16 | }
17 | \description{
18 | Remove unzipped monthly data folders
19 | }
20 | \references{
21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 | }
23 | \author{
24 | Christine Laney \email{claney@battelleecology.org}
25 | }
26 | \keyword{internal}
27 |
--------------------------------------------------------------------------------
/neonUtilities/man/convByteSize.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/convByteSize.R
3 | \name{convByteSize}
4 | \alias{convByteSize}
5 | \title{Convert a number of bytes into megabytes or gigabytes}
6 | \usage{
7 | convByteSize(objSize)
8 | }
9 | \arguments{
10 | \item{objSize}{The size in bytes}
11 | }
12 | \value{
13 | The size of the file in megabytes or gigabytes
14 | }
15 | \description{
16 | For any number of bytes, convert to a number of MB or GB
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Claire Lunch \email{clunch@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/datasetQuery.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/datasetQuery.R
3 | \name{datasetQuery}
4 | \alias{datasetQuery}
5 | \title{Query the query endpoint of the NEON API and create an arrow dataset from the results}
6 | \usage{
7 | datasetQuery(
8 | dpID,
9 | site = "all",
10 | startdate = NA,
11 | enddate = NA,
12 | tabl = NA,
13 | hor = NA,
14 | ver = NA,
15 | package = "basic",
16 | release = "current",
17 | include.provisional = FALSE,
18 | token = NA_character_
19 | )
20 | }
21 | \arguments{
22 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001}
23 |
24 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.}
25 |
26 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
27 |
28 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
29 |
30 | \item{tabl}{The name of a single data table to download.}
31 |
32 | \item{hor}{The horizontal index of data to download. Only applicable to sensor (IS) data.}
33 |
34 | \item{ver}{The vertical index of data to download. Only applicable to sensor (IS) data.}
35 |
36 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.}
37 |
38 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.}
39 |
40 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.}
41 |
42 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.}
43 | }
44 | \value{
45 | An arrow dataset for the data requested.
46 | }
47 | \description{
48 | Uses the query endpoint of the NEON API to find the full list of files for a given data product, release, site(s), and date range, then turns them into an arrow dataset.
49 | }
50 | \references{
51 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
52 | }
53 | \author{
54 | Claire Lunch \email{clunch@battelleecology.org}
55 | }
56 |
--------------------------------------------------------------------------------
/neonUtilities/man/dateConvert.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/dateConvert.R
3 | \name{dateConvert}
4 | \alias{dateConvert}
5 | \title{Convert date stamps from character}
6 | \usage{
7 | dateConvert(dates, useFasttime = FALSE)
8 | }
9 | \arguments{
10 | \item{dates}{A vector of date values in character format [character]}
11 |
12 | \item{useFasttime}{Should the fasttime package be used for date conversion? Defaults to false. [logical]}
13 | }
14 | \value{
15 | A POSIXct vector, if possible; if conversion was unsuccessful, the original vector is returned
16 | }
17 | \description{
18 | Attempt to convert date stamps from character, iterating through known NEON date formats
19 | }
20 | \references{
21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 | }
23 | \author{
24 | Claire Lunch \email{clunch@battelleecology.org}
25 | }
26 | \keyword{internal}
27 |
--------------------------------------------------------------------------------
/neonUtilities/man/eddyStampCheck.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/eddyStampCheck.R
3 | \name{eddyStampCheck}
4 | \alias{eddyStampCheck}
5 | \title{Convert date stamps from character and check for only one record in a day}
6 | \usage{
7 | eddyStampCheck(tab, useFasttime = FALSE)
8 | }
9 | \arguments{
10 | \item{tab}{A table of SAE data}
11 |
12 | \item{useFasttime}{Should the fasttime package be used to convert time stamps?}
13 | }
14 | \value{
15 | The same table of SAE data, with time stamps converted and empty records representing a single day (filler records inserted during processing) removed.
16 | }
17 | \description{
18 | Convert SAE time stamps to POSIX and check for missing data
19 | }
20 | \references{
21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 | }
23 | \author{
24 | Claire Lunch \email{clunch@battelleecology.org}
25 | }
26 | \keyword{internal}
27 |
--------------------------------------------------------------------------------
/neonUtilities/man/findDatatables.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/findDatatables.R
3 | \name{findDatatables}
4 | \alias{findDatatables}
5 | \title{Find data tables}
6 | \usage{
7 | findDatatables(folder, fnames = T)
8 | }
9 | \arguments{
10 | \item{folder}{The folder of the outputs}
11 |
12 | \item{fnames}{Full names - if true, then return the full file names including enclosing folders, if false, return only the file names}
13 | }
14 | \value{
15 | a data frame of file names
16 | }
17 | \description{
18 | List the names of the data tables within each folder
19 | }
20 | \references{
21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 | }
23 | \author{
24 | Christine Laney \email{claney@battelleecology.org}
25 | }
26 | \keyword{internal}
27 |
--------------------------------------------------------------------------------
/neonUtilities/man/findTablesByFormat.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/findTablesByFormat.R
3 | \name{findTablesByFormat}
4 | \alias{findTablesByFormat}
5 | \title{Find unique data tables in dataset}
6 | \usage{
7 | findTablesByFormat(datatables)
8 | }
9 | \arguments{
10 | \item{datatables}{A list of data files}
11 | }
12 | \value{
13 | An array of unique table names and their types
14 | }
15 | \description{
16 | Find the unique data tables that are present in the dataset (e.g., 2 minute vs 30 minute, or pinning vs identification data) and their types, based on the file name formatting. Adapted from findTablesUnique().
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Claire Lunch \email{clunch@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/findTablesUnique.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/findTablesUnique.R
3 | \name{findTablesUnique}
4 | \alias{findTablesUnique}
5 | \title{Find unique data tables in dataset}
6 | \usage{
7 | findTablesUnique(datatables, tabletypes)
8 | }
9 | \arguments{
10 | \item{datatables}{A list of data files}
11 | }
12 | \value{
13 | An array of unique table names
14 | }
15 | \description{
16 | Find the unique data tables that are present in the dataset (e.g., 2 minute vs 30 minute, or pinning vs identification data)
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Christine Laney \email{claney@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/footRaster.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/footRaster.R
3 | \name{footRaster}
4 | \alias{footRaster}
5 | \title{Extract eddy covariance footprint data from HDF5 format}
6 | \usage{
7 | footRaster(filepath)
8 | }
9 | \arguments{
10 | \item{filepath}{One of: a folder containing NEON EC H5 files, a zip file of DP4.00200.001 data downloaded from the NEON data portal, a folder of DP4.00200.001 data downloaded by the neonUtilities::zipsByProduct() function, or a single NEON EC H5 file. Filepath can only contain files for a single site. [character]}
11 | }
12 | \value{
13 | A rasterStack object containing all the footprints in the input files, plus one layer (the first in the stack) containing the mean footprint.
14 | }
15 | \description{
16 | Create a raster of flux footprint data. Specific to expanded package of eddy covariance data product: DP4.00200.001
17 | For definition of a footprint, see Glossary of Meteorology: https://glossary.ametsoc.org/wiki/Footprint
18 | For background information about flux footprints and considerations around the time scale of footprint calculations, see Amiro 1998: https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.922.4124&rep=rep1&type=pdf
19 | }
20 | \details{
21 | Given a filepath containing H5 files of expanded package DP4.00200.001 data, extracts flux footprint data and creates a raster.
22 | }
23 | \examples{
24 | \dontrun{
25 | # To run the function on a zip file downloaded from the NEON data portal:
26 | ftprnt <- footRaster(filepath="~/NEON_eddy-flux.zip")
27 | }
28 | }
29 | \references{
30 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
31 | }
32 | \author{
33 | Claire Lunch \email{clunch@battelleecology.org}
34 | }
35 |
--------------------------------------------------------------------------------
/neonUtilities/man/getAPI.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getAPI.R
3 | \name{getAPI}
4 | \alias{getAPI}
5 | \title{Get the data from API}
6 | \usage{
7 | getAPI(apiURL, token = NA_character_)
8 | }
9 | \arguments{
10 | \item{apiURL}{The API endpoint URL}
11 |
12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.}
13 | }
14 | \description{
15 | Accesses the API with options to use the user-specific API token generated within data.neonscience.org user accounts.
16 | }
17 | \references{
18 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
19 | }
20 | \author{
21 | Nate Mietkiewicz \email{mietkiewicz@battelleecology.org}
22 | }
23 | \keyword{internal}
24 |
--------------------------------------------------------------------------------
/neonUtilities/man/getAPIHeaders.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getAPIHeaders.R
3 | \name{getAPIHeaders}
4 | \alias{getAPIHeaders}
5 | \title{Get only headers from API}
6 | \usage{
7 | getAPIHeaders(apiURL, token = NA_character_)
8 | }
9 | \arguments{
10 | \item{apiURL}{The API endpoint URL}
11 |
12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.}
13 | }
14 | \description{
15 | Accesses the API with options to use the user-specific API token generated within neon.datascience user accounts.
16 | }
17 | \references{
18 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
19 | }
20 | \author{
21 | Claire Lunch \email{clunch@battelleecology.org}
22 | }
23 | \keyword{internal}
24 |
--------------------------------------------------------------------------------
/neonUtilities/man/getAttributes.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getAttributes.R
3 | \name{getAttributes}
4 | \alias{getAttributes}
5 | \title{Extract attributes from eddy covariance H5 files}
6 | \usage{
7 | getAttributes(fil, sit, type, valName)
8 | }
9 | \arguments{
10 | \item{fil}{File path to the H5 file to extract attributes from [character]}
11 |
12 | \item{sit}{The site, for site attributes. Must match site of file path. [character]}
13 |
14 | \item{type}{The type of attributes to retrieve. [character]}
15 |
16 | \item{valName}{If CO2 validation metadata are requested, the H5 name of the level where they can be found. [character]}
17 | }
18 | \value{
19 | A data frame with one row containing the extracted attributes
20 | }
21 | \description{
22 | Extract attribute metadata from H5 files
23 | }
24 | \references{
25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
26 | }
27 | \author{
28 | Claire Lunch \email{clunch@battelleecology.org}
29 | }
30 | \keyword{internal}
31 |
--------------------------------------------------------------------------------
/neonUtilities/man/getAvg.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getAvg.R
3 | \name{getAvg}
4 | \alias{getAvg}
5 | \title{Get a list of the available averaging intervals for a data product}
6 | \usage{
7 | getAvg(dpID, token = NA_character_)
8 | }
9 | \arguments{
10 | \item{dpID}{The identifier of the NEON data product, in the form DPL.PRNUM.REV, e.g. DP1.00006.001}
11 |
12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
13 | }
14 | \value{
15 | A vector of the available averaging intervals, typically in minutes.
16 | }
17 | \description{
18 | Most IS products are available at multiple averaging intervals; get a list of what's available for a given data product
19 | }
20 | \examples{
21 | # Get available averaging intervals for PAR data product
22 | getAvg("DP1.00024.001")
23 | }
24 | \references{
25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
26 | }
27 | \author{
28 | Claire Lunch \email{clunch@battelleecology.org}
29 | }
30 |
--------------------------------------------------------------------------------
/neonUtilities/man/getCitation.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getCitation.R
3 | \name{getCitation}
4 | \alias{getCitation}
5 | \title{Get a Bibtex citation for NEON data with a DOI, or generate a provisional Bibtex citation}
6 | \usage{
7 | getCitation(dpID = NA_character_, release = NA_character_)
8 | }
9 | \arguments{
10 | \item{dpID}{The data product ID of the data to be cited [character]}
11 |
12 | \item{release}{The data release to be cited. Can be provisional. [character]}
13 | }
14 | \value{
15 | A character string containing the Bibtex citation
16 | }
17 | \description{
18 | Use the DOI Foundation API to get Bibtex-formatted citations for NEON data, or use a template to generate a Bibtex citation for provisional data. Helper function to download and stacking functions.
19 | }
20 | \examples{
21 | \dontrun{
22 | # Get the citation for Breeding landbird point counts (DP1.10003.001), RELEASE-2023
23 | cit <- getCitation(dpID="DP1.10003.001", release="RELEASE-2023")
24 | }
25 | }
26 | \references{
27 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
28 | }
29 | \author{
30 | Claire Lunch \email{clunch@battelleecology.org}
31 | }
32 |
--------------------------------------------------------------------------------
/neonUtilities/man/getDatatable.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getDatatable.R
3 | \name{getDatatable}
4 | \alias{getDatatable}
5 | \title{Get NEON data table}
6 | \usage{
7 | getDatatable(
8 | dpid = NA,
9 | data_table_name = NA,
10 | sample_location_list = NA,
11 | sample_location_type = "siteID",
12 | sample_date_min = "2012-01-01",
13 | sample_date_max = Sys.Date(),
14 | sample_date_format = "\%Y-\%m-\%d",
15 | data_package_type = "basic",
16 | url_prefix_data = "https://data.neonscience.org/api/v0/data/",
17 | url_prefix_products = "https://data.neonscience.org/api/v0/products/",
18 | token = NA_character_
19 | )
20 | }
21 | \arguments{
22 | \item{dpid}{character sting for NEON data product ID}
23 |
24 | \item{data_table_name}{character sting for name of the data table to download, e.g., 'sls_soilCoreCollection'}
25 |
26 | \item{sample_location_list}{list of sites, domains, etc. If NA, retrieve all data for the given data table / dpid combination.}
27 |
28 | \item{sample_location_type}{character sting for location type, such as 'siteID'. Must be one of the NEON controlled terms. If you're unsure, use 'siteID'}
29 |
30 | \item{sample_date_min}{start date for query. Default is 1-Jan-2012, and this should capture the earliest NEON data record.}
31 |
32 | \item{sample_date_max}{end date for query. Default is current date.}
33 |
34 | \item{sample_date_format}{date format. Default/expected format is yyyy-mm-dd}
35 |
36 | \item{data_package_type}{package type, either 'basic' or 'expanded'. If unsure, use 'expanded'}
37 |
38 | \item{url_prefix_data}{data endpoint for NEON API.}
39 |
40 | \item{url_prefix_products}{products endpoint for NEON API.}
41 |
42 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
43 | }
44 | \value{
45 | data frame with selected NEON data
46 | }
47 | \description{
48 | This is a function to retrieve a data table
49 | from the NEON data portal for sites and dates provided by the
50 | enduser. NOTE that this function only works for NEON
51 | Observation System (OS) data products, and only for select tables
52 | }
53 | \references{
54 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
55 | }
56 | \author{
57 | Eric R. Sokol \email{esokol@battelleecology.org}
58 | }
59 |
--------------------------------------------------------------------------------
/neonUtilities/man/getEddyLog.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getEddyLog.R
3 | \name{getEddyLog}
4 | \alias{getEddyLog}
5 | \title{Get the full issue log set for the SAE bundle}
6 | \usage{
7 | getEddyLog(token = NA_character_)
8 | }
9 | \arguments{
10 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
11 | }
12 | \value{
13 | A table of issues reported for the data product.
14 | }
15 | \description{
16 | Use the NEON API to get the issue log from all products in the bundle in a user-friendly format
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Claire Lunch \email{clunch@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/getFileUrls.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getFileUrls.R
3 | \name{getFileUrls}
4 | \alias{getFileUrls}
5 | \title{Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame}
6 | \usage{
7 | getFileUrls(m.urls, include.provisional, token = NA)
8 | }
9 | \arguments{
10 | \item{m.urls}{The monthly API URL for the AOP files}
11 |
12 | \item{include.provisional}{T or F, should provisional data be included in downloaded files?}
13 |
14 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
15 | }
16 | \value{
17 | A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0)
18 | }
19 | \description{
20 | Used to generate a data frame of available AOP files.
21 | }
22 | \references{
23 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
24 | }
25 | \author{
26 | Claire Lunch \email{clunch@battelleecology.org}
27 | Christine Laney \email{claney@battelleecology.org}
28 | }
29 | \keyword{internal}
30 |
--------------------------------------------------------------------------------
/neonUtilities/man/getIssueLog.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getIssueLog.R
3 | \name{getIssueLog}
4 | \alias{getIssueLog}
5 | \title{Get the issue log for a specific data product}
6 | \usage{
7 | getIssueLog(dpID = NA, token = NA_character_)
8 | }
9 | \arguments{
10 | \item{dpID}{The data product identifier, formatted as DP#.#####.###}
11 |
12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
13 | }
14 | \value{
15 | A table of issues reported for the data product.
16 | }
17 | \description{
18 | Use the NEON API to get the issue log in a user-friendly format
19 | }
20 | \examples{
21 | # Get documentation and availability of plant foliar properties data product
22 | cfcIssues <- getIssueLog("DP1.10026.001")
23 | }
24 | \references{
25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
26 | }
27 | \author{
28 | Claire Lunch \email{clunch@battelleecology.org}
29 | }
30 |
--------------------------------------------------------------------------------
/neonUtilities/man/getNeonDOI.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getNeonDOI.R
3 | \name{getNeonDOI}
4 | \alias{getNeonDOI}
5 | \title{Get either a list of NEON DOIs, or the DOI for a specific data product and release}
6 | \usage{
7 | getNeonDOI(dpID = NA_character_, release = NA_character_)
8 | }
9 | \arguments{
10 | \item{dpID}{The data product identifier, formatted as DP#.#####.### [character]}
11 |
12 | \item{release}{Name of a specific release, e.g. RELEASE-2022 [character]}
13 | }
14 | \value{
15 | A table of data product IDs and DOIs.
16 | }
17 | \description{
18 | Use the DataCite API to get NEON data DOIs in a user-friendly format
19 | }
20 | \examples{
21 | \dontrun{
22 | # Get all NEON data DOIs
23 | allDOIs <- getNeonDOI()
24 | }
25 | }
26 | \references{
27 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
28 | }
29 | \author{
30 | Claire Lunch \email{clunch@battelleecology.org}
31 | }
32 |
--------------------------------------------------------------------------------
/neonUtilities/man/getPackage.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getPackage.R
3 | \name{getPackage}
4 | \alias{getPackage}
5 | \title{Get NEON data package}
6 | \usage{
7 | getPackage(dpID, site_code, year_month, package = "basic", savepath = getwd())
8 | }
9 | \arguments{
10 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001}
11 |
12 | \item{site_code}{A four-letter NEON research site code, such as HEAL for Healy.}
13 |
14 | \item{year_month}{The year and month of interest, in format YYYY-MM.}
15 |
16 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.}
17 |
18 | \item{savepath}{The location to save the output files to}
19 | }
20 | \value{
21 | A zipped monthly file
22 | }
23 | \description{
24 | Get a zipped file for a single data product, site, and year-month combination. Use the NEON data portal or API to determine data
25 | availability by data product, site, and year-month combinations.
26 | }
27 | \references{
28 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
29 | }
30 | \author{
31 | Christine Laney \email{claney@battelleecology.org}
32 | }
33 |
--------------------------------------------------------------------------------
/neonUtilities/man/getProductInfo.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getProductInfo.R
3 | \name{getProductInfo}
4 | \alias{getProductInfo}
5 | \title{Get NEON data product information}
6 | \usage{
7 | getProductInfo(dpID = "", token = NA)
8 | }
9 | \arguments{
10 | \item{dpID}{The data product id (optional), formated as DP#.#####.###}
11 |
12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
13 | }
14 | \value{
15 | A named list of metadata and availability information for a single data product. If the dpID argument is omitted, a table of information for all data products in the NEON catalog.
16 | }
17 | \description{
18 | Use the NEON API to get data product information such as availability, science team, etc.
19 | }
20 | \examples{
21 | # Get documentation and availability of plant foliar properties data product
22 | cfcInfo <- getProductInfo("DP1.10026.001")
23 | }
24 | \references{
25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
26 | }
27 | \author{
28 | Christine Laney \email{claney@battelleecology.org}
29 | }
30 |
--------------------------------------------------------------------------------
/neonUtilities/man/getProductSensors.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getProductSensors.R
3 | \name{getProductSensors}
4 | \alias{getProductSensors}
5 | \title{Get data product-sensor relationships}
6 | \usage{
7 | getProductSensors()
8 | }
9 | \value{
10 | A data frame
11 | }
12 | \description{
13 | Pull all data from the NEON API /products endpoint, create a data frame with data product ID, data product name, and sensor type.
14 | }
15 | \examples{
16 | \dontrun{
17 | sensors <- getProductSensors()
18 | }
19 | }
20 | \references{
21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 | }
23 | \author{
24 | Christine Laney \email{claney@battelleecology.org}
25 | }
26 |
--------------------------------------------------------------------------------
/neonUtilities/man/getReadmePublicationDate.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getReadmePublicationDate.R
3 | \name{getReadmePublicationDate}
4 | \alias{getReadmePublicationDate}
5 | \title{Scrape the publication date from each ReadMe file}
6 | \usage{
7 | getReadmePublicationDate(savepath, out_filepath, dpID)
8 | }
9 | \arguments{
10 | \item{savepath}{The root folder directory where the ReadMe files are located.}
11 |
12 | \item{out_filepath}{The output directory and filename.}
13 |
14 | \item{dpID}{The data product identifier}
15 | }
16 | \description{
17 | Given a directory, this will recursively list all of the ReadMe files that were unzipped.
18 | This should result in a single text file with a list of all of the publication dates from the ReadMe file.
19 | }
20 | \references{
21 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
22 | }
23 | \author{
24 | Nathan Mietkiewicz \email{mietkiewicz@battelleecology.org}
25 | }
26 | \keyword{internal}
27 |
--------------------------------------------------------------------------------
/neonUtilities/man/getRecentPublication.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getRecentPublication.R
3 | \name{getRecentPublication}
4 | \alias{getRecentPublication}
5 | \title{Returns the most recent files for those that do not need stacking}
6 | \usage{
7 | getRecentPublication(inList)
8 | }
9 | \arguments{
10 | \item{inList}{The list of files.}
11 | }
12 | \description{
13 | Given a list of files, this will order and return the file with the most recent publication date.
14 | }
15 | \references{
16 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
17 | }
18 | \author{
19 | Nathan Mietkiewicz \email{mietkiewicz@battelleecology.org}
20 | }
21 | \keyword{internal}
22 |
--------------------------------------------------------------------------------
/neonUtilities/man/getTaxonTable.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getTaxonTable.R
3 | \name{getTaxonTable}
4 | \alias{getTaxonTable}
5 | \title{Get NEON taxon table}
6 | \usage{
7 | getTaxonTable(
8 | taxonType = NA,
9 | recordReturnLimit = NA,
10 | stream = "true",
11 | token = NA
12 | )
13 | }
14 | \arguments{
15 | \item{taxonType}{Character string for the taxonTypeCode. Must be one of ALGAE, BEETLE, BIRD,
16 | FISH, HERPETOLOGY, MACROINVERTEBRATE, MOSQUITO, MOSQUITO_PATHOGENS, SMALL_MAMMAL, PLANT, TICK}
17 |
18 | \item{recordReturnLimit}{Integer. The number of items to limit the result set to. If NA, will return all records in table.}
19 |
20 | \item{stream}{Character string, true or false. Option to obtain the result as a stream. Utilize for large requests.}
21 |
22 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
23 | }
24 | \value{
25 | data frame with selected NEON data
26 | }
27 | \description{
28 | This is a function to retrieve a taxon table
29 | from the NEON data portal for the taxon type by the
30 | enduser.
31 | }
32 | \references{
33 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
34 | }
35 | \author{
36 | Eric R. Sokol \email{esokol@battelleecology.org}
37 | }
38 |
--------------------------------------------------------------------------------
/neonUtilities/man/getTileUrls.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getTileUrls.R
3 | \name{getTileUrls}
4 | \alias{getTileUrls}
5 | \title{Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame}
6 | \usage{
7 | getTileUrls(
8 | m.urls,
9 | tileEasting,
10 | tileNorthing,
11 | include.provisional,
12 | token = NA_character_
13 | )
14 | }
15 | \arguments{
16 | \item{m.urls}{The monthly API URL for the AOP tile.}
17 |
18 | \item{tileEasting}{A vector containing the easting UTM coordinates of the locations to download.}
19 |
20 | \item{tileNorthing}{A vector containing the northing UTM coordinates of the locations to download.}
21 |
22 | \item{include.provisional}{T or F, should provisional data be included in downloaded files?}
23 |
24 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.}
25 | }
26 | \value{
27 | A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0)
28 | }
29 | \description{
30 | Produces a data frame that is populated by available tiles for the AOP product.
31 | }
32 | \references{
33 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
34 | }
35 | \author{
36 | Claire Lunch \email{clunch@battelleecology.org}
37 | Christine Laney \email{claney@battelleecology.org}
38 | }
39 | \keyword{internal}
40 |
--------------------------------------------------------------------------------
/neonUtilities/man/getTimeIndex.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getTimeIndex.R
3 | \name{getTimeIndex}
4 | \alias{getTimeIndex}
5 | \title{Get a list of the available time intervals for a data product}
6 | \usage{
7 | getTimeIndex(dpID, token = NA_character_)
8 | }
9 | \arguments{
10 | \item{dpID}{The identifier of the NEON data product, in the form DPL.PRNUM.REV, e.g. DP1.00006.001}
11 |
12 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
13 | }
14 | \value{
15 | A vector of the available time intervals, typically in minutes.
16 | }
17 | \description{
18 | Most IS products are available at multiple time intervals; get a list of what's available for a given data product
19 | }
20 | \examples{
21 | # Get available time intervals for PAR data product
22 | getTimeIndex("DP1.00024.001")
23 | }
24 | \references{
25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
26 | }
27 | \author{
28 | Claire Lunch \email{clunch@battelleecology.org}
29 | }
30 |
--------------------------------------------------------------------------------
/neonUtilities/man/getTitle.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getTitle.R
3 | \name{getTitle}
4 | \alias{getTitle}
5 | \title{Get NEON data product title}
6 | \usage{
7 | getTitle(filename)
8 | }
9 | \arguments{
10 | \item{filename}{A NEON file name}
11 | }
12 | \value{
13 | A title for the respective GeoCSV file
14 | }
15 | \description{
16 | Create a title for a NEON data CSV file
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Christine Laney \email{claney@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/getVariables.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getVariables.R
3 | \name{getVariables}
4 | \alias{getVariables}
5 | \title{Get correct data types}
6 | \usage{
7 | getVariables(varFile)
8 | }
9 | \arguments{
10 | \item{varFile}{A file that contains variable definitions}
11 | }
12 | \value{
13 | A data frame with fieldName and assigned column class, along with table if present
14 | }
15 | \description{
16 | Support way to force R to read assign correct data types to each column based on variables file
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Christine Laney \email{claney@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/getVariablesEddy.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getVariablesEddy.R
3 | \name{getVariablesEddy}
4 | \alias{getVariablesEddy}
5 | \title{Get variable names and units from SAE H5 files}
6 | \usage{
7 | getVariablesEddy(tabList)
8 | }
9 | \arguments{
10 | \item{tabList}{A list of SAE data tables}
11 | }
12 | \value{
13 | A table of variable names and units, aggregated from the input tables
14 | }
15 | \description{
16 | Extract variable names and units from SAE H5 files and return in user-friendly form. Used in stackEddy(), not intended for independent use.
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Claire Lunch \email{clunch@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/getVarsEddy.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getVarsEddy.R
3 | \name{getVarsEddy}
4 | \alias{getVarsEddy}
5 | \title{Extract list of eddy covariance tables from HDF5 files}
6 | \usage{
7 | getVarsEddy(filepath)
8 | }
9 | \arguments{
10 | \item{filepath}{The folder containing the H5 file [character]}
11 | }
12 | \value{
13 | A data frame of the metadata for each data table in the HDF5 file
14 | }
15 | \description{
16 | Extracts a list of table metadata from a single HDF5 file. Specific to eddy covariance data product: DP4.00200.001. Can inform inputs to stackEddy(); variables listed in 'name' are available inputs to the 'var' parameter in stackEddy().
17 | }
18 | \examples{
19 | \dontrun{
20 | # read variables from a file in a hypothetical filepath
21 | ec.vars <- getVarsEddy(filepath='/data/NEON.D19.BONA.DP4.00200.001.nsae.2017-12.basic.h5')
22 | }
23 | }
24 | \references{
25 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
26 | }
27 | \author{
28 | Claire Lunch \email{clunch@battelleecology.org}
29 | }
30 |
--------------------------------------------------------------------------------
/neonUtilities/man/getZipUrls.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/getZipUrls.R
3 | \name{getZipUrls}
4 | \alias{getZipUrls}
5 | \title{Get and store the file names, S3 URLs, file size, and download status (default = 0) in a data frame}
6 | \usage{
7 | getZipUrls(
8 | month.urls,
9 | avg,
10 | package,
11 | dpID,
12 | release,
13 | tabl,
14 | include.provisional,
15 | token = NA_character_
16 | )
17 | }
18 | \arguments{
19 | \item{month.urls}{The monthly API URL for the URL files}
20 |
21 | \item{avg}{Global variable for averaging interval}
22 |
23 | \item{package}{Global varaible for package type (basic or expanded)}
24 |
25 | \item{dpID}{Global variable for data product ID}
26 |
27 | \item{release}{Data release to be downloaded}
28 |
29 | \item{tabl}{Table name to get}
30 |
31 | \item{include.provisional}{Should provisional data be included?}
32 |
33 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
34 | }
35 | \value{
36 | A dataframe comprised of file names, S3 URLs, file size, and download status (default = 0)
37 | }
38 | \description{
39 | Used to generate a data frame of available zipfile URLs.
40 | }
41 | \references{
42 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
43 | }
44 | \author{
45 | Claire Lunch \email{clunch@battelleecology.org}
46 | Christine Laney \email{claney@battelleecology.org}
47 | }
48 | \keyword{internal}
49 |
--------------------------------------------------------------------------------
/neonUtilities/man/listFilesInZip.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/listFilesInZip.R
3 | \name{listFilesInZip}
4 | \alias{listFilesInZip}
5 | \title{Get a data frame with the names of all files within a zipped NEON data package}
6 | \usage{
7 | listFilesInZip(zippath)
8 | }
9 | \arguments{
10 | \item{zippath}{The path to a zip file}
11 | }
12 | \value{
13 | A list of filenames within the given zip file
14 | }
15 | \description{
16 | Given the top level zip file, return dataframe of all of the files within it without
17 | unzipping the file
18 | }
19 | \references{
20 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
21 | }
22 | \author{
23 | Christine Laney \email{claney@battelleecology.org}
24 | }
25 | \keyword{internal}
26 |
--------------------------------------------------------------------------------
/neonUtilities/man/listZipfiles.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/listZipfiles.R
3 | \name{listZipfiles}
4 | \alias{listZipfiles}
5 | \title{Get all zip file names within a zipped NEON data package}
6 | \usage{
7 | listZipfiles(zippath)
8 | }
9 | \arguments{
10 | \item{zippath}{The path to a zip file}
11 | }
12 | \value{
13 | An array of all zip files contained within the focal zip file
14 | }
15 | \description{
16 | Given the data frame of all the files within the top level zip file,
17 | return an array of just the zip file names (no pdf, xml, or other files).
18 | }
19 | \references{
20 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
21 | }
22 | \author{
23 | Christine Laney \email{claney@battelleecology.org}
24 | }
25 | \keyword{internal}
26 |
--------------------------------------------------------------------------------
/neonUtilities/man/loadByProduct.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/loadByProduct.R
3 | \name{loadByProduct}
4 | \alias{loadByProduct}
5 | \title{Get files from NEON API, stack tables, and load into the current environment}
6 | \usage{
7 | loadByProduct(
8 | dpID,
9 | site = "all",
10 | startdate = NA,
11 | enddate = NA,
12 | package = "basic",
13 | release = "current",
14 | timeIndex = "all",
15 | tabl = "all",
16 | check.size = TRUE,
17 | include.provisional = FALSE,
18 | nCores = 1,
19 | forceParallel = FALSE,
20 | token = NA_character_,
21 | useFasttime = FALSE,
22 | avg = NA
23 | )
24 | }
25 | \arguments{
26 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001}
27 |
28 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.}
29 |
30 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
31 |
32 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
33 |
34 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.}
35 |
36 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.}
37 |
38 | \item{timeIndex}{Either the string 'all', or the time index of data to download, in minutes. Only applicable to sensor (IS) data. Defaults to 'all'.}
39 |
40 | \item{tabl}{Either the string 'all', or the name of a single data table to download. Defaults to 'all'.}
41 |
42 | \item{check.size}{T or F, should the user approve the total file size before downloading? Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.}
43 |
44 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.}
45 |
46 | \item{nCores}{The number of cores to parallelize the stacking procedure. By default it is set to a single core.}
47 |
48 | \item{forceParallel}{If the data volume to be processed does not meet minimum requirements to run in parallel, this overrides. Set to FALSE as default.}
49 |
50 | \item{token}{User specific API token (generated within data.neonscience.org user accounts)}
51 |
52 | \item{useFasttime}{Should the fasttime package be used to read date-time fields? Defaults to false.}
53 |
54 | \item{avg}{Deprecated; use timeIndex}
55 | }
56 | \value{
57 | A named list of all the data tables in the data product downloaded, plus a validation file and a variables file, as available.
58 | }
59 | \description{
60 | Pull files from the NEON API, by data product, merge data for each table, and read into the current R environment
61 | }
62 | \details{
63 | All available data meeting the query criteria will be downloaded. Most data products are collected at only a subset of sites, and dates of collection vary. Consult the NEON data portal for sampling details.
64 | Dates are specified only to the month because NEON data are provided in monthly packages. Any month included in the search criteria will be included in the download. Start and end date are inclusive.
65 | }
66 | \examples{
67 | \dontrun{
68 | # To download plant foliar properties data from all sites, expanded data package:
69 | cfc <- loadByProduct(dpID="DP1.10026.001", site="all", package="expanded")
70 | }
71 | }
72 | \references{
73 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
74 | }
75 | \author{
76 | Claire Lunch \email{clunch@battelleecology.org}
77 | }
78 |
--------------------------------------------------------------------------------
/neonUtilities/man/makePosColumns.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/makePosColumns.R
3 | \name{makePosColumns}
4 | \alias{makePosColumns}
5 | \title{Create position (horizontal and vertical) columns}
6 | \usage{
7 | makePosColumns(d, datafl, site)
8 | }
9 | \arguments{
10 | \item{d}{A data table}
11 |
12 | \item{datafl}{A data file name}
13 | }
14 | \value{
15 | A data table with new columns
16 | }
17 | \description{
18 | For instrumented meteorological data products, create position (horizontal and vertical) columns based on values
19 | embedded in the file names.
20 | }
21 | \references{
22 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
23 | }
24 | \author{
25 | Christine Laney \email{claney@battelleecology.org}
26 | }
27 | \keyword{internal}
28 |
--------------------------------------------------------------------------------
/neonUtilities/man/other_bundles.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/other_bundles.R
3 | \docType{data}
4 | \name{other_bundles}
5 | \alias{other_bundles}
6 | \title{Bundled vegetation and sediment data product information}
7 | \format{
8 | A data frame with 2 variables:
9 | \describe{
10 | \item{product}{Data product ID of a product}
11 | \item{homeProduct}{Data product ID of the corresponding home data product}
12 | }
13 | }
14 | \source{
15 | NEON data product bundles
16 | }
17 | \usage{
18 | other_bundles
19 | }
20 | \description{
21 | A dataset containing NEON data product codes of vegetation and sediment data products and the "home" data products they are bundled with.
22 | }
23 | \keyword{datasets}
24 |
--------------------------------------------------------------------------------
/neonUtilities/man/queryFiles.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/queryFiles.R
3 | \name{queryFiles}
4 | \alias{queryFiles}
5 | \title{Get a list of data files from the query endpoint of the NEON API}
6 | \usage{
7 | queryFiles(
8 | dpID,
9 | site = "all",
10 | startdate = NA,
11 | enddate = NA,
12 | package = "basic",
13 | release = "current",
14 | timeIndex = "all",
15 | tabl = "all",
16 | metadata = TRUE,
17 | include.provisional = FALSE,
18 | token = NA_character_
19 | )
20 | }
21 | \arguments{
22 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001}
23 |
24 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.}
25 |
26 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
27 |
28 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
29 |
30 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.}
31 |
32 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.}
33 |
34 | \item{timeIndex}{Either the string 'all', or the time index of data to download, in minutes. Only applicable to sensor (IS) data. Defaults to 'all'.}
35 |
36 | \item{tabl}{Either the string 'all', or the name of a single data table to download. Defaults to 'all'.}
37 |
38 | \item{metadata}{T or F, should urls for metadata files (variables, sensor positions, etc) be included. Defaults to F, can only be set to T if tabl is not 'all'.}
39 |
40 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.}
41 |
42 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.}
43 | }
44 | \value{
45 | A list of two elements: (1) the set of urls matching the query; (2) the most recent variables file for the set of urls
46 | }
47 | \description{
48 | Uses the query endpoint of the NEON API to find the full list of files for a given data product, release, site(s), and date range.
49 | }
50 | \references{
51 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
52 | }
53 | \author{
54 | Claire Lunch \email{clunch@battelleecology.org}
55 | }
56 |
--------------------------------------------------------------------------------
/neonUtilities/man/quietMessages.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/quietMessages.R
3 | \name{quietMessages}
4 | \alias{quietMessages}
5 | \title{Will suppress all output messages, while retaining the output dataframe}
6 | \usage{
7 | quietMessages(toBeQuieted)
8 | }
9 | \arguments{
10 | \item{toBeQuieted}{Input to be quieted}
11 | }
12 | \value{
13 | The expected output without associated messages/warnings.
14 | }
15 | \description{
16 | Used to quiet all output messages
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Nate Mietkiewicz \email{mietkiewicz@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/readTableNEON.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/readTableNEON.R
3 | \name{readTableNEON}
4 | \alias{readTableNEON}
5 | \title{Read a NEON data table with correct data types for each variable}
6 | \usage{
7 | readTableNEON(dataFile, varFile, useFasttime = FALSE)
8 | }
9 | \arguments{
10 | \item{dataFile}{A data frame containing a NEON data table, or the filepath to a data table to load}
11 |
12 | \item{varFile}{A data frame containing the corresponding NEON variables file, or the filepath to the variables file}
13 |
14 | \item{useFasttime}{Should the fasttime package be used to read date-time variables? Defaults to false.}
15 | }
16 | \value{
17 | A data frame of a NEON data table, with column classes assigned by data type
18 | }
19 | \description{
20 | Load a table into R, assigning classes to each column based on data types in variables file; or convert a table already loaded
21 | }
22 | \references{
23 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
24 | }
25 | \author{
26 | Claire Lunch \email{clunch@battelleecology.org}
27 | }
28 |
--------------------------------------------------------------------------------
/neonUtilities/man/schemaAllStrings.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/schemaAllStrings.R
3 | \name{schemaAllStrings}
4 | \alias{schemaAllStrings}
5 | \title{Create an arrow schema with every variable coded as a string field.}
6 | \usage{
7 | schemaAllStrings(variables)
8 | }
9 | \arguments{
10 | \item{variables}{A data frame containing a NEON variables file for a single table, or a set of field names.}
11 | }
12 | \value{
13 | An arrow schema for the relevant data table with all variables set to string.
14 | }
15 | \description{
16 | Use the field names in a NEON variables file to create an arrow schema of all strings, or, if no variables file is available, read the header and assign everything as string.
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Claire Lunch \email{clunch@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/schemaFromVar.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/schemaFromVar.R
3 | \name{schemaFromVar}
4 | \alias{schemaFromVar}
5 | \title{Create an arrow schema from a NEON variables file.}
6 | \usage{
7 | schemaFromVar(variables, tab, package)
8 | }
9 | \arguments{
10 | \item{variables}{A data frame containing a NEON variables file, or a url pointing to a NEON variables file.}
11 |
12 | \item{tab}{The name of the table to generate a schema from.}
13 |
14 | \item{package}{Should the schema be created for the basic or expanded package?}
15 | }
16 | \value{
17 | An arrow schema for the relevant data table.
18 | }
19 | \description{
20 | Use the field names and data types in a NEON variables file to create an arrow schema.
21 | }
22 | \references{
23 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
24 | }
25 | \author{
26 | Claire Lunch \email{clunch@battelleecology.org}
27 | }
28 |
--------------------------------------------------------------------------------
/neonUtilities/man/shared_aquatic.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/shared_aquatic.R
3 | \docType{data}
4 | \name{shared_aquatic}
5 | \alias{shared_aquatic}
6 | \title{Terrestrial-aquatic shared data information}
7 | \format{
8 | A data frame with 3 variables:
9 | \describe{
10 | \item{site}{site code of a NEON aquatic site}
11 | \item{towerSite}{site code of the NEON terrestrial site used as the data source for the corresponding aquatic site}
12 | \item{product}{Data product ID of the data products to which the corresponding terrestrial-aquatic site relationship relates}
13 | }
14 | }
15 | \source{
16 | NEON site layouts and spatial design
17 | }
18 | \usage{
19 | shared_aquatic
20 | }
21 | \description{
22 | A dataset containing NEON site codes and data product IDs for places where meteorological data from terrestrial sites are used as the data of record for nearby aquatic sites as well.
23 | }
24 | \keyword{datasets}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/shared_flights.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/shared_flights.R
3 | \docType{data}
4 | \name{shared_flights}
5 | \alias{shared_flights}
6 | \title{Flight coverage information}
7 | \format{
8 | A data frame with 2 variables:
9 | \describe{
10 | \item{site}{site code of a NEON site}
11 | \item{flightSite}{site code that matches the file naming for flights that may include "site"}
12 | }
13 | }
14 | \source{
15 | NEON flight plans
16 | }
17 | \usage{
18 | shared_flights
19 | }
20 | \description{
21 | A dataset containing NEON site codes for places where a single AOP flight may cover multiple sites
22 | }
23 | \keyword{datasets}
24 |
--------------------------------------------------------------------------------
/neonUtilities/man/stackByTable.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/stackByTable.R
3 | \name{stackByTable}
4 | \alias{stackByTable}
5 | \title{Join data files in a zipped NEON data package by table type}
6 | \usage{
7 | stackByTable(
8 | filepath,
9 | savepath = NA,
10 | folder = FALSE,
11 | saveUnzippedFiles = FALSE,
12 | dpID = NA,
13 | package = NA,
14 | nCores = 1,
15 | useFasttime = FALSE
16 | )
17 | }
18 | \arguments{
19 | \item{filepath}{The location of the zip file}
20 |
21 | \item{savepath}{The location to save the output files to}
22 |
23 | \item{folder}{T or F: does the filepath point to a parent, unzipped folder, or a zip file? If F, assumes the filepath points to a zip file. Defaults to F. No longer needed; included for back compatibility.}
24 |
25 | \item{saveUnzippedFiles}{T or F: should the unzipped monthly data folders be retained?}
26 |
27 | \item{dpID}{Data product ID of product to stack. Ignored and determined from data unless input is a vector of files, generally via stackFromStore().}
28 |
29 | \item{package}{Data download package, either basic or expanded. Ignored and determined from data unless input is a vector of files, generally via stackFromStore().}
30 |
31 | \item{nCores}{The number of cores to parallelize the stacking procedure. To automatically use the maximum number of cores on your machine we suggest setting nCores=parallel::detectCores(). By default it is set to a single core.}
32 |
33 | \item{useFasttime}{Should the fasttime package be used to read date-time variables? Only relevant if savepath="envt". Defaults to false.}
34 | }
35 | \value{
36 | All files are unzipped and one file for each table type is created and written. If savepath="envt" is specified, output is a named list of tables; otherwise, function output is null and files are saved to the location specified.
37 | }
38 | \description{
39 | Given a zipped data file, do a full join of all data files, grouped by table type.
40 | This should result in a small number of large files.
41 | }
42 | \examples{
43 | \dontrun{
44 | # To unzip and merge files downloaded from the NEON Data Portal
45 | stackByTable("~/NEON_par.zip")
46 |
47 | # To unzip and merge files downloaded using zipsByProduct()
48 | stackByTable("~/filesToStack00024")
49 | }
50 | }
51 | \references{
52 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
53 | }
54 | \author{
55 | Christine Laney \email{claney@battelleecology.org}
56 | Claire Lunch \email{clunch@battelleecology.org}
57 | }
58 |
--------------------------------------------------------------------------------
/neonUtilities/man/stackDataFilesParallel.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/stackDataFilesParallel.R
3 | \name{stackDataFilesParallel}
4 | \alias{stackDataFilesParallel}
5 | \title{Join data files in a unzipped NEON data package by table type}
6 | \usage{
7 | stackDataFilesParallel(folder, nCores = 1, dpID)
8 | }
9 | \arguments{
10 | \item{folder}{The location of the data}
11 |
12 | \item{nCores}{The number of cores to parallelize the stacking procedure. To automatically use the maximum number of cores on your machine we suggest setting 'nCores=parallel::detectCores()'. By default it is set to a single core. If the files are less than 25000 bytes the userdefined nCores will be overridden to a single core.}
13 |
14 | \item{dpID}{The data product identifier}
15 | }
16 | \value{
17 | One file for each table type is created and written.
18 | }
19 | \description{
20 | Given a folder of unzipped files (unzipped NEON data file), do a full join of all data files, grouped by table type.
21 | This should result in a small number of large files.
22 | }
23 | \references{
24 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
25 | }
26 | \author{
27 | Christine Laney \email{claney@battelleecology.org}
28 | }
29 | \keyword{internal}
30 |
--------------------------------------------------------------------------------
/neonUtilities/man/stackEddy.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/stackEddy.R
3 | \name{stackEddy}
4 | \alias{stackEddy}
5 | \title{Extract eddy covariance data from HDF5 format}
6 | \usage{
7 | stackEddy(
8 | filepath,
9 | level = "dp04",
10 | var = NA,
11 | avg = NA,
12 | metadata = FALSE,
13 | useFasttime = FALSE,
14 | runLocal = FALSE
15 | )
16 | }
17 | \arguments{
18 | \item{filepath}{One of: a folder containing NEON EC H5 files, a zip file of DP4.00200.001 data downloaded from the NEON data portal, a folder of DP4.00200.001 data downloaded by the neonUtilities::zipsByProduct() function, or a single NEON EC H5 file [character]}
19 |
20 | \item{level}{The level of data to extract; one of dp01, dp02, dp03, dp04 [character]}
21 |
22 | \item{var}{The variable set to extract. Can be any of the variables in the "name" level or the "system" level of the H5 file; use the getVarsEddy() function to see the available variables. From the inputs, all variables from "name" and all variables from "system" will be returned, but if variables from both "name" and "system" are specified, the function will return only the intersecting set. This allows the user to, e.g., return only the pressure data ("pres") from the CO2 storage system ("co2Stor"), instead of all the pressure data from all instruments. [character]}
23 |
24 | \item{avg}{The averaging interval to extract, in minutes [numeric]}
25 |
26 | \item{metadata}{Should the output include metadata from the attributes of the H5 files? Defaults to false. Even when false, variable definitions, issue logs, and science review flags will be included. [logical]}
27 |
28 | \item{useFasttime}{Should the fasttime package be used to convert time stamps to time format? Decreases stacking time but can introduce imprecision at the millisecond level. Defaults to false. [logical]}
29 |
30 | \item{runLocal}{Set to TRUE to omit any calls to the NEON API. Data are extracted and reformatted from local files, but citation and issue log are not retrieved. [logical]}
31 | }
32 | \value{
33 | A named list of data frames. One data frame per site, plus one data frame containing the metadata (objDesc) table and one data frame containing units for each variable (variables).
34 | }
35 | \description{
36 | Convert data of choice from HDF5 to tabular format. Specific to eddy covariance data product: DP4.00200.001
37 | }
38 | \details{
39 | Given a filepath containing H5 files of DP4.00200.001 data, extracts variables, stacks data tables over time, and joins variables into a single table.
40 | For data product levels 2-4 (dp02, dp03, dp04), joins all available data, except for the flux footprint data in the expanded package.
41 | For dp01, an averaging interval and a set of variable names must be provided as inputs.
42 | }
43 | \examples{
44 | \dontrun{
45 | # To extract and merge Level 4 data tables, where data files are in the working directory
46 | flux <- stackEddy(filepath=getwd(), level='dp04', var=NA, avg=NA)
47 | }
48 | }
49 | \references{
50 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
51 | }
52 | \author{
53 | Claire Lunch \email{clunch@battelleecology.org}
54 | }
55 |
--------------------------------------------------------------------------------
/neonUtilities/man/stackFromStore.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/stackFromStore.R
3 | \name{stackFromStore}
4 | \alias{stackFromStore}
5 | \title{Select files from a stored set of NEON data, created by neonstore package methods or another method}
6 | \usage{
7 | stackFromStore(
8 | filepaths,
9 | dpID,
10 | site = "all",
11 | startdate = NA,
12 | enddate = NA,
13 | pubdate = NA,
14 | timeIndex = "all",
15 | level = "dp04",
16 | var = NA,
17 | zipped = FALSE,
18 | package = "basic",
19 | load = TRUE,
20 | nCores = 1
21 | )
22 | }
23 | \arguments{
24 | \item{filepaths}{Either a vector of filepaths pointing to files to be stacked, or a single directory containing files that can be stacked, with selection criteria detmined by the other inputs. In both cases files to be stacked must be either site-month zip files or unzipped folders corresponding to site-month zips. [character]}
25 |
26 | \item{dpID}{The NEON data product ID of the data to be stacked [character]}
27 |
28 | \item{site}{Either "all" or a vector of NEON site codes to be stacked [character]}
29 |
30 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA. [character]}
31 |
32 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA. [character]}
33 |
34 | \item{pubdate}{The maximum publication date of data to include in stacking, in the form YYYY-MM-DD. If NA, the most recently published data for each product-site-month combination will be selected. Otherwise, the most recent publication date that is older than pubdate will be selected. Thus the data stacked will be the data that would have been accessed on the NEON Data Portal, if it had been downloaded on pubdate. [character]}
35 |
36 | \item{timeIndex}{Either the string 'all', or the time index of data to be stacked, in minutes. Only applicable to sensor (IS) and eddy covariance data. Defaults to 'all'. [character]}
37 |
38 | \item{level}{Data product level of data to stack. Only applicable to eddy covariance (SAE) data; see stackEddy() documentation. [character]}
39 |
40 | \item{var}{Variables to be extracted and stacked from H5 files. Only applicable to eddy covariance (SAE) data; see stackEddy() documentation. [character]}
41 |
42 | \item{zipped}{Should stacking use data from zipped files or unzipped folders? This option allows zips and their equivalent unzipped folders to be stored in the same directory; stacking will extract whichever is specified. Defaults to FALSE, i.e. stacking using unzipped folders. [logical]}
43 |
44 | \item{package}{Either "basic" or "expanded", indicating which data package to stack. Defaults to basic. [character]}
45 |
46 | \item{load}{If TRUE, stacked data are read into the current R environment. If FALSE, stacked data are written to the directory where data files are stored. Defaults to TRUE. [logical]}
47 |
48 | \item{nCores}{Number of cores to use for optional parallel processing. Defaults to 1. [integer]}
49 | }
50 | \value{
51 | If load=TRUE, returns a named list of stacked data tables. If load=FALSE, return is empty and stacked files are written to data directory.
52 | }
53 | \description{
54 | Select files from a stored set based on input criteria and pass to stackByTable() or stackEddy()
55 | }
56 | \references{
57 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
58 | }
59 | \author{
60 | Claire Lunch \email{clunch@battelleecology.org}
61 | }
62 |
--------------------------------------------------------------------------------
/neonUtilities/man/table_types.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/table_types.R
3 | \docType{data}
4 | \name{table_types}
5 | \alias{table_types}
6 | \title{Publication table information}
7 | \format{
8 | A data frame with 5 variables. Number of rows changes frequently as more tables are added:
9 | \describe{
10 | \item{productID}{data product ID}
11 | \item{tableName}{name of table}
12 | \item{tableDesc}{description of table}
13 | \item{tableType}{type of table (important for knowing which tables to stack, and how to stack)}
14 | \item{tableTMI}{a time index (e.g., 0 = native resolution, 1 = 1 minute, 30 = 30 minute averages or totals)}
15 | }
16 | }
17 | \source{
18 | NEON database
19 | }
20 | \usage{
21 | table_types
22 | }
23 | \description{
24 | A dataset containing publication table names, descriptions, type (site-date, site-all, lab-all, lab-current), and a time index
25 | }
26 | \keyword{datasets}
27 |
--------------------------------------------------------------------------------
/neonUtilities/man/timeStampSet.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/timeStampSet.R
3 | \name{timeStampSet}
4 | \alias{timeStampSet}
5 | \title{Generate a consensus set of time stamps from a set of input tables.}
6 | \usage{
7 | timeStampSet(tabList)
8 | }
9 | \arguments{
10 | \item{tabList}{A list of SAE data tables}
11 | }
12 | \value{
13 | A table of time stamps (start and end times) aggregated from the input tables
14 | }
15 | \description{
16 | Generate consensus SAE time stamps from a set of tables. Used in stackEddy(), not intended for independent use.
17 | }
18 | \references{
19 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
20 | }
21 | \author{
22 | Claire Lunch \email{clunch@battelleecology.org}
23 | }
24 | \keyword{internal}
25 |
--------------------------------------------------------------------------------
/neonUtilities/man/transformFileToGeoCSV.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/transformFileToGeoCSV.R
3 | \name{transformFileToGeoCSV}
4 | \alias{transformFileToGeoCSV}
5 | \title{Transform NEON CSV file to GeoCSV}
6 | \usage{
7 | transformFileToGeoCSV(infile, varfile, outfile)
8 | }
9 | \arguments{
10 | \item{infile}{The path to the file that needs to be parsed}
11 |
12 | \item{varfile}{The path to the variables file needed to parse the infile}
13 |
14 | \item{outfile}{The path where the new GeoCSV file should be placed}
15 | }
16 | \value{
17 | The same data file with a GeoCSV header
18 | }
19 | \description{
20 | Read in a NEON monthly data zip file and parse the respective variables file to create a new GeoCSV file
21 | }
22 | \references{
23 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
24 | }
25 | \author{
26 | Christine Laney \email{claney@battelleecology.org}
27 | }
28 |
--------------------------------------------------------------------------------
/neonUtilities/man/unzipZipfileParallel.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/unzipZipfileParallel.R
3 | \name{unzipZipfileParallel}
4 | \alias{unzipZipfileParallel}
5 | \title{Unzip a zip file either at just the top level or recursively through the file}
6 | \usage{
7 | unzipZipfileParallel(
8 | zippath,
9 | outpath = substr(zippath, 1, nchar(zippath) - 4),
10 | level = "all",
11 | nCores = 1
12 | )
13 | }
14 | \arguments{
15 | \item{zippath}{The filepath of the input file}
16 |
17 | \item{outpath}{The name of the folder to save unpacked files to}
18 |
19 | \item{level}{Whether the unzipping should occur only for the 'top' zip file, or unzip 'all' recursively, or only files 'in' the folder specified}
20 |
21 | \item{nCores}{Number of cores to use for parallelization}
22 | }
23 | \description{
24 | Unzip a zip file either at just the top level or recursively through the file
25 | }
26 | \references{
27 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
28 | }
29 | \author{
30 | Christine Laney \email{claney@battelleecology.org}
31 | Claire Lunch \email{clunch@battelleecology.org}
32 | }
33 | \keyword{internal}
34 |
--------------------------------------------------------------------------------
/neonUtilities/man/zipsByProduct.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/zipsByProduct.R
3 | \name{zipsByProduct}
4 | \alias{zipsByProduct}
5 | \title{Get files from NEON API to feed the stackByTable() function}
6 | \usage{
7 | zipsByProduct(
8 | dpID,
9 | site = "all",
10 | startdate = NA,
11 | enddate = NA,
12 | package = "basic",
13 | release = "current",
14 | timeIndex = "all",
15 | tabl = "all",
16 | check.size = TRUE,
17 | include.provisional = FALSE,
18 | savepath = NA,
19 | load = F,
20 | token = NA_character_,
21 | avg = NA
22 | )
23 | }
24 | \arguments{
25 | \item{dpID}{The identifier of the NEON data product to pull, in the form DPL.PRNUM.REV, e.g. DP1.10023.001}
26 |
27 | \item{site}{Either the string 'all', meaning all available sites, or a character vector of 4-letter NEON site codes, e.g. c('ONAQ','RMNP'). Defaults to all.}
28 |
29 | \item{startdate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
30 |
31 | \item{enddate}{Either NA, meaning all available dates, or a character vector in the form YYYY-MM, e.g. 2017-01. Defaults to NA.}
32 |
33 | \item{package}{Either 'basic' or 'expanded', indicating which data package to download. Defaults to basic.}
34 |
35 | \item{release}{The data release to be downloaded; either 'current' or the name of a release, e.g. 'RELEASE-2021'. 'current' returns the most recent release, as well as provisional data if include.provisional is set to TRUE. To download only provisional data, use release='PROVISIONAL'. Defaults to 'current'.}
36 |
37 | \item{timeIndex}{Either the string 'all', or the time index of data to download, in minutes. Only applicable to sensor (IS) data. Defaults to 'all'.}
38 |
39 | \item{tabl}{Either the string 'all', or the name of a single data table to download. Defaults to 'all'.}
40 |
41 | \item{check.size}{T or F, should the user approve the total file size before downloading? Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.}
42 |
43 | \item{include.provisional}{T or F, should provisional data be included in downloaded files? Defaults to F. See https://www.neonscience.org/data-samples/data-management/data-revisions-releases for details on the difference between provisional and released data.}
44 |
45 | \item{savepath}{The location to save the output files to}
46 |
47 | \item{load}{T or F, are files saved locally or loaded directly? Used silently with loadByProduct(), do not set manually.}
48 |
49 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.}
50 |
51 | \item{avg}{Deprecated; use timeIndex}
52 | }
53 | \value{
54 | A folder in the working directory (or in savepath, if specified), containing all zip files meeting query criteria.
55 | }
56 | \description{
57 | Pull files from the NEON API, by data product, in a structure that will allow them to be stacked by the stackByTable() function
58 | }
59 | \details{
60 | All available data meeting the query criteria will be downloaded. Most data products are collected at only a subset of sites, and dates of collection vary. Consult the NEON data portal for sampling details.
61 | Dates are specified only to the month because NEON data are provided in monthly packages. Any month included in the search criteria will be included in the download. Start and end date are inclusive.
62 | timeIndex: NEON sensor data are published at pre-determined averaging intervals, usually 1 and 30 minutes. The default download includes all available data. Download volume can be greatly reduced by downloading only the 30 minute files, if higher frequency data are not needed. Use getTimeIndex() to find the available averaging intervals for each sensor data product.
63 | }
64 | \examples{
65 | \dontrun{
66 | # To download plant foliar properties data from all sites, expanded data package:
67 | zipsByProduct(dpID="DP1.10026.001", site="all", package="expanded")
68 | }
69 | }
70 | \references{
71 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
72 | }
73 | \author{
74 | Claire Lunch \email{clunch@battelleecology.org}
75 | }
76 |
--------------------------------------------------------------------------------
/neonUtilities/man/zipsByURI.Rd:
--------------------------------------------------------------------------------
1 | % Generated by roxygen2: do not edit by hand
2 | % Please edit documentation in R/zipsByURI.R
3 | \name{zipsByURI}
4 | \alias{zipsByURI}
5 | \title{Get files from NEON ECS Bucket using URLs in stacked data}
6 | \usage{
7 | zipsByURI(
8 | filepath,
9 | savepath = paste0(filepath, "/ECS_zipFiles"),
10 | pick.files = FALSE,
11 | check.size = TRUE,
12 | unzip = TRUE,
13 | saveZippedFiles = FALSE,
14 | token = NA_character_
15 | )
16 | }
17 | \arguments{
18 | \item{filepath}{The location of the NEON data containing URIs. Can be either a local directory containing NEON tabular data or a list object containing tabular data.}
19 |
20 | \item{savepath}{The location to save the output files from the ECS bucket, optional.
21 | Defaults to creating a "ECS_zipFiles" folder in the filepath directory.}
22 |
23 | \item{pick.files}{T or F, should the user be told the name of each file before downloading?
24 | Defaults to F. When working in batch mode, or other non-interactive workflow, use pick.files=F.}
25 |
26 | \item{check.size}{T or F, should the user be told the total file size before downloading?
27 | Defaults to T. When working in batch mode, or other non-interactive workflow, use check.size=F.}
28 |
29 | \item{unzip}{T or F, indicates if the downloaded zip files from ECS buckets should be
30 | unzipped into the same directory, defaults to T. Supports .zip and .tar.gz files currently.}
31 |
32 | \item{saveZippedFiles}{T or F: should the zip files be retained after unzipping? Defaults to F.}
33 |
34 | \item{token}{User specific API token (generated within data.neonscience.org user accounts). Optional.}
35 | }
36 | \value{
37 | A folder in the working directory (or in savepath, if specified), containing all files meeting query criteria.
38 | }
39 | \description{
40 | Read in a set of URLs from NEON data tables and then download the data from the NEON ECS buckets.
41 | Assumes data tables are in the format resulting from merging files using stackByTable().
42 | File downloads from ECS can be extremely large; be prepared for long download times and large file storage.
43 | }
44 | \examples{
45 | \dontrun{
46 | # To download stream morphology data from stacked data:
47 | zipsByURI(filepath="~/filesToStack00131/stackedFiles")
48 | }
49 | }
50 | \references{
51 | License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
52 | }
53 | \author{
54 | Kaelin Cawley \email{kcawley@battelleecology.org}
55 | }
56 |
--------------------------------------------------------------------------------
/neonUtilities/neonUtilities.Rproj:
--------------------------------------------------------------------------------
1 | Version: 1.0
2 |
3 | RestoreWorkspace: Default
4 | SaveWorkspace: Default
5 | AlwaysSaveHistory: Default
6 |
7 | EnableCodeIndexing: Yes
8 | UseSpacesForTab: Yes
9 | NumSpacesForTab: 2
10 | Encoding: UTF-8
11 |
12 | RnwWeave: Sweave
13 | LaTeX: pdfLaTeX
14 |
15 | AutoAppendNewline: Yes
16 | StripTrailingWhitespace: Yes
17 |
18 | BuildType: Package
19 | PackageUseDevtools: Yes
20 | PackageInstallArgs: --no-multiarch --with-keep.source
21 |
--------------------------------------------------------------------------------
/neonUtilities/tests/testthat.R:
--------------------------------------------------------------------------------
1 | library(testthat)
2 | library(neonUtilities)
3 |
4 | test_check("neonUtilities")
5 |
--------------------------------------------------------------------------------
/neonUtilities/tests/testthat/test_byTileAOP.R:
--------------------------------------------------------------------------------
1 | context("byTileAOP")
2 | library(neonUtilities)
3 |
4 | test_that("Test that unequal length easting and northing vectors return error", {
5 | expect_error(byTileAOP(dpID="DP3.30015.001", site="WREF", year=2017,
6 | easting=571000, northing=c(5079000,5080000)),
7 | "Easting and northing vector lengths do not match, and/or contain null values. Cannot identify paired coordinates.")
8 | })
9 |
--------------------------------------------------------------------------------
/neonUtilities/tests/testthat/test_stackByTable.R:
--------------------------------------------------------------------------------
1 | context("Table stacking")
2 | library(neonUtilities)
3 |
4 | files_match <- function(object, output, expected) {
5 | object
6 | comp <- compare(output$uid,
7 | expected$uid)
8 | expect(
9 | comp$equal,
10 | sprintf("Output files do not match previous version files.\n%s", comp$message)
11 | )
12 | }
13 |
14 | f <- stackByTable(system.file("extdata", "NEON_gp.zip",
15 | package="neonUtilities"), savepath="envt")
16 | test_that("Test that correct files are created for microbe group abundances", {
17 | files_match({},
18 | f$mga_soilGroupAbundances,
19 | read.csv(system.file("extdata", "expected/gp/mga_soilGroupAbundances.csv",
20 | package="neonUtilities")))
21 | })
22 |
23 |
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.0.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.0.0.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.0.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.0.1.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.2.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.2.0.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.2.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.2.1.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.2.2.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.2.2.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.0.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.1.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.2.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.2.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.3.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.3.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.4.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.4.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.5.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.5.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.6.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.6.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.7.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.7.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.8.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.8.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_1.3.9.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_1.3.9.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.0.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.0.0.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.0.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.0.1.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.1.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.0.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.1.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.1.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.1.2.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.2.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.1.3.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.3.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.1.4.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.1.4.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.2.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.2.0.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.2.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.2.1.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.3.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.3.0.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.4.0.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.4.0.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.4.1.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.4.1.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.4.2.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.4.2.tar.gz
--------------------------------------------------------------------------------
/release_archive/neonUtilities_2.4.3.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/NEONScience/NEON-utilities/f77a1e70cc62a88add88f1d14d6fc8ad4234c8ae/release_archive/neonUtilities_2.4.3.tar.gz
--------------------------------------------------------------------------------