diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index f4b17a4..08087ee 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -14,13 +14,19 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes + EDI_API_KEY: ${{ secrets.EDI_API_KEY }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - uses: r-lib/actions/setup-r@v2 with: use-public-rspm: true + - name: Create and populate .Renviron file + run: | + echo EDI_API_KEY="$EDI_API_KEY" >> ~/.Renviron + shell: bash + - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: any::rcmdcheck diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index f190d51..3f19b6c 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -13,14 +13,20 @@ jobs: runs-on: ubuntu-latest env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + EDI_API_KEY: ${{ secrets.EDI_API_KEY }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - uses: r-lib/actions/setup-r@v2 with: use-public-rspm: true + - name: Create and populate .Renviron file + run: | + echo EDI_API_KEY="$EDI_API_KEY" >> ~/.Renviron + shell: bash + - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: any::covr diff --git a/DESCRIPTION b/DESCRIPTION index 7918a4e..d4b85a6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -46,13 +46,15 @@ Imports: stats, utils, wql, - rvest + rvest, + EDIutils URL: https://github.com/InteragencyEcologicalProgram/zooper BugReports: https://github.com/InteragencyEcologicalProgram/zooper/issues -RoxygenNote: 7.3.3 Suggests: spelling, testthat (>= 2.1.0), - covr + covr, + withr Language: en-US Config/testthat/edition: 3 +Config/roxygen2/version: 8.1.0 diff --git a/NEWS.md b/NEWS.md index e127eb7..c27d82f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ # zooper (development version) +- Updated download method to use `EDIutils` package to handle new requirement for API keys. Added a function to check for an access key and provide instructions if one is not present. +- Removed parameter for `Redownload_data` from `Zoopdownloader` and `reload_data` from `Zoopsynther`, because the EDIutils functions directly read the data but do not save it. - Added EMP and FRP data released in May 2026 - Added EMP data released in June 2025 (sampling from 1972–2023), **which no longer includes amphipods in the macro dataset**; see the [amphipods_notes (EDI 522)](https://portal.edirepository.org/nis/mapbrowse?scope=edi&identifier=522) for more information. - ~~Fixed issue where EMP macro amphipod counts from before 2014 were being converted to unidentified mysids instead of unidentified amphipods, since amphipods were not identified to species before 2014.~~ diff --git a/R/Download_utilities.R b/R/Download_utilities.R index ab6f304..cffbdb5 100644 --- a/R/Download_utilities.R +++ b/R/Download_utilities.R @@ -56,6 +56,23 @@ html_file_list<-function(URL){ return(files) } +#' Create a list of entities from an EDI package +#' +#' This function lists all data entities for an EDI package +#' +#' @param PID EDI package ID +#' @keywords internal +#' + +edi_entity_list<-function(PID){ + + latest_revision <- paste("edi", PID, max(EDIutils::list_data_package_revisions("edi", PID)), sep=".") + entities <- EDIutils::read_data_entity_names(packageId = latest_revision) + entities<-purrr::set_names(c(entities$entityId, latest_revision), c(entities$entityName, "PID")) + + return(entities) +} + #' Extract latest EDI files #' This function extracts the latest version of a zooplankton EDI package and the list of files from that package #' @@ -72,39 +89,35 @@ zoop_urls<-function(Sources){ 'EMP', 'FMWT', 'STN','20mm', 'FRP', 'YBFMP', 'DOP'") } + if(any(Sources%in%c("EMP", "STN", "FMWT", "YBFMP", "DOP", "FRP"))){ + check_EDI_cred() + } + out<-list() if("EMP"%in%Sources){ - EMP_revision_url <- "https://pasta.lternet.edu/package/eml/edi/522" - EMP_latest_revision <- utils::tail(Tryer(n=3, fun=readLines, con=EMP_revision_url, warn = FALSE), 1) - EMP_pkg_url <- paste0("https://pasta.lternet.edu/package/data/eml/edi/522/", EMP_latest_revision) - EMP_entities <- Tryer(n=3, fun=readLines, con=EMP_pkg_url, warn = FALSE) - EMP_name_urls <- paste("https://pasta.lternet.edu/package/name/eml/edi/522", EMP_latest_revision, EMP_entities, sep="/") - names(EMP_entities) <- purrr::map_chr(EMP_name_urls, ~Tryer(n=3, fun=readLines, con=.x, warn = FALSE)) - - out$EMP$Meso<-paste0(EMP_pkg_url, "/", EMP_entities["1972-2024_CBMatrix_EDI"]) - out$EMP$Micro<-paste0(EMP_pkg_url, "/", EMP_entities["1972-2024_PumpMatrix_EDI"]) - out$EMP$Macro<-paste0(EMP_pkg_url, "/", EMP_entities["1972-2024_MysidMatrix_EDI"]) - out$EMP$Lengths<-paste0(EMP_pkg_url, "/", EMP_entities["1972-2024_Mysid_Length_Data_EDI"]) + EMP_entities<-edi_entity_list(522) + out$EMP$Meso<-EMP_entities[grep("CBMatrix", names(EMP_entities))] + out$EMP$Micro<-EMP_entities[grep("PumpMatrix", names(EMP_entities))] + out$EMP$Macro<-EMP_entities[grep("MysidMatrix", names(EMP_entities))] + out$EMP$Lengths<-EMP_entities[grep("Mysid_Length_Data", names(EMP_entities))] + out$EMP$PID<-EMP_entities["PID"] } if(any(c("STN", "FMWT")%in%Sources)){ - FMWTSTN_revision_url <- "https://pasta.lternet.edu/package/eml/edi/1103" - FMWTSTN_latest_revision <- utils::tail(Tryer(n=3, fun=readLines, con=FMWTSTN_revision_url, warn = FALSE), 1) - FMWTSTN_pkg_url <- paste0("https://pasta.lternet.edu/package/data/eml/edi/1103/", FMWTSTN_latest_revision) - FMWTSTN_entities <- Tryer(n=3, fun=readLines, con=FMWTSTN_pkg_url, warn = FALSE) - FMWTSTN_name_urls <- paste("https://pasta.lternet.edu/package/name/eml/edi/1103", FMWTSTN_latest_revision, FMWTSTN_entities, sep="/") - names(FMWTSTN_entities) <- purrr::map_chr(FMWTSTN_name_urls, ~Tryer(n=3, fun=readLines, con=.x, warn = FALSE)) + FMWTSTN_entities<-edi_entity_list(1103) SMSCG_URL<-"https://filelib.wildlife.ca.gov/Public/TownetFallMidwaterTrawl/Zooplankton_SMSCG/" SMSCG_files<-html_file_list(SMSCG_URL) - out$FMWTSTN$Meso<-paste0(FMWTSTN_pkg_url, "/", FMWTSTN_entities["FMWT_STN_CBNetCPUE.csv"]) + out$FMWTSTN$Meso<-FMWTSTN_entities[grep("FMWT_STN_CBNet", names(FMWTSTN_entities))] out$SMSCG$Meso<-SMSCG_files[grep("CBNet", SMSCG_files)] - out$FMWTSTN$Macro<-paste0(FMWTSTN_pkg_url, "/", FMWTSTN_entities["FMWT_MysidNetCPUE.csv"]) + out$FMWTSTN$Macro<-FMWTSTN_entities[grep("MysidNetCPUE", names(FMWTSTN_entities))] out$SMSCG$Macro<-SMSCG_files[grep("MysidNet", SMSCG_files)] + + out$FMWTSTN$PID<-FMWTSTN_entities["PID"] } if("20mm"%in%Sources){ @@ -115,50 +128,63 @@ zoop_urls<-function(Sources){ } if("YBFMP"%in%Sources){ - YBFMP_revision_url <- "https://pasta.lternet.edu/package/eml/edi/494" - YBFMP_latest_revision <- utils::tail(Tryer(n=3, fun=readLines, con=YBFMP_revision_url, warn = FALSE), 1) - YBFMP_pkg_url <- paste0("https://pasta.lternet.edu/package/data/eml/edi/494/", YBFMP_latest_revision) - YBFMP_entities <- Tryer(n=3, fun=readLines, con=YBFMP_pkg_url, warn = FALSE) - YBFMP_name_urls <- paste("https://pasta.lternet.edu/package/name/eml/edi/494", YBFMP_latest_revision, YBFMP_entities, sep="/") - names(YBFMP_entities) <- purrr::map_chr(YBFMP_name_urls, ~Tryer(n=3, fun=readLines, con=.x, warn = FALSE)) + YBFMP_entities<-edi_entity_list(494) + + out$YBFMP$Meso<-YBFMP_entities[grep("Zooplankton Data", names(YBFMP_entities))] - out$YBFMP<-paste0(YBFMP_pkg_url, "/", YBFMP_entities["Zooplankton Data"]) + out$YBFMP$PID<-YBFMP_entities["PID"] } if("DOP"%in%Sources){ - DOP_revision_url <- "https://pasta.lternet.edu/package/eml/edi/1187" - DOP_latest_revision <- utils::tail(Tryer(n=3, fun=readLines, con=DOP_revision_url, warn = FALSE), 1) - DOP_pkg_url <- paste0("https://pasta.lternet.edu/package/data/eml/edi/1187/", DOP_latest_revision) - DOP_entities <- Tryer(n=3, fun=readLines, con=DOP_pkg_url, warn = FALSE) - DOP_name_urls <- paste("https://pasta.lternet.edu/package/name/eml/edi/1187", DOP_latest_revision, DOP_entities, sep="/") - names(DOP_entities) <- purrr::map_chr(DOP_name_urls, ~Tryer(n=3, fun=readLines, con=.x, warn = FALSE)) + DOP_entities<-edi_entity_list(1187) - out$DOP$Meso<-paste0(DOP_pkg_url, "/", DOP_entities[grep("Mesozooplankton_Abundance", names(DOP_entities))]) + out$DOP$Meso<-DOP_entities[grep("Mesozooplankton_Abundance", names(DOP_entities))] - out$DOP$Macro<-paste0(DOP_pkg_url, "/", DOP_entities[grep("Macrozooplankton_Abundance", names(DOP_entities))]) + out$DOP$Macro<-DOP_entities[grep("Macrozooplankton_Abundance", names(DOP_entities))] - out$DOP$trawls<-paste0(DOP_pkg_url, "/", DOP_entities[grep("TowData", names(DOP_entities))]) + out$DOP$trawls<-DOP_entities[grep("TowData", names(DOP_entities))] + + out$DOP$PID<-DOP_entities["PID"] } if("FRP"%in%Sources){ - FRP_revision_url <- "https://pasta.lternet.edu/package/eml/edi/269" - FRP_latest_revision <- utils::tail(Tryer(n=3, fun=readLines, con=FRP_revision_url, warn = FALSE), 1) - FRP_pkg_url <- paste0("https://pasta.lternet.edu/package/data/eml/edi/269/", FRP_latest_revision) - FRP_entities <- Tryer(n=3, fun=readLines, con=FRP_pkg_url, warn = FALSE) - FRP_name_urls <- paste("https://pasta.lternet.edu/package/name/eml/edi/269", FRP_latest_revision, FRP_entities, sep="/") - names(FRP_entities) <- purrr::map_chr(FRP_name_urls, ~Tryer(n=3, fun=readLines, con=.x, warn = FALSE)) + FRP_entities<-edi_entity_list(269) + + out$FRP$Meso<-FRP_entities[grep("zoops_FRP", names(FRP_entities))] - out$FRP$Meso<-paste0(FRP_pkg_url, "/", FRP_entities[grep("zoops_FRP", names(FRP_entities))]) + out$FRP$Macro<-FRP_entities[grep("macroinvert_FRP", names(FRP_entities))] - out$FRP$Macro<-paste0(FRP_pkg_url, "/", FRP_entities[grep("macroinvert_FRP", names(FRP_entities))]) + out$FRP$site<-FRP_entities[grep("sitevisit_FRP", names(FRP_entities))] - out$FRP$site<-paste0(FRP_pkg_url, "/", FRP_entities[grep("sitevisit_FRP", names(FRP_entities))]) + out$FRP$PID<-FRP_entities["PID"] } return(out) } +#' check for EDI credentials +#' +#' Check if EDI credentials are properly set, and return a message if not +#' +#' @noRd + +check_EDI_cred<-function(){ + #borrowed from https://github.com/ropensci/EDIutils/blob/main/tests/testthat/helper-test-package.R + # Identical to function in deltafish + has_token <- (Sys.getenv("EDI_TOKEN") != "") && (Sys.getenv("EDI_TOKEN") != "foobar") + has_key <- (Sys.getenv("EDI_API_KEY") != "") && (Sys.getenv("EDI_API_KEY") != "foobar") + + if(!has_token && !has_key){ + stop("An EDI API key is now required by EDI. To resolve this:\n", + "1) Please log in or create an account at https://auth.edirepository.org/ \n", + "2) Access your API key from your profile -> Access Keys.\n", + "3) You can then use the key either with EDIutils::login() ", + "or by adding it to your .Renviron, e.g., by running usethis::edit_r_environ() ", + "and, adding a line `EDI_API_KEY=YOUR_KEY`, saving, and restarting R.") + } +} + #set options to allow access to EDI without signing in #options(HTTPUserAgent="EDI_CodeGen") diff --git a/R/biomass.R b/R/biomass.R index edec0c3..8e45581 100644 --- a/R/biomass.R +++ b/R/biomass.R @@ -2,12 +2,12 @@ #' This function converts zooplankton CPUE to carbon biomass (Carbon biomass per unit effort (\eqn{\mu}g/ \ifelse{html}{\out{m3}}{\eqn{m^{3}}})) for taxa with conversion equations. #' #' @param Zoop Zooplankton count dataset -#' @param ZoopLengths Zooplankton length dataset for macrozooplankton. +#' @param ZoopLengths Zooplankton length dataset for macrozooplankton. May be left NULL if the Macro size class is not included. #' @param Biomass_mesomicro The micro and meso zooplankton biomass conversion table. The default is \code{\link{biomass_mesomicro}} #' @param Biomass_macro The macro zooplankton biomass conversion table. The default is \code{\link{biomass_macro}} Zoopbiomass<-function(Zoop, - ZoopLengths, + ZoopLengths=NULL, Biomass_mesomicro=zooper::biomass_mesomicro, Biomass_macro=zooper::biomass_macro) { @@ -23,6 +23,10 @@ Zoopbiomass<-function(Zoop, message("Note that macro zooplankton biomass conversion is currently only available for EMP, so macrozooplankton biomass will not be returned for other surveys") } + if ("Macro"%in%Size_class & is.null(ZoopLengths)){ + stop("Parameter ZoopLengths must be included for Macrozooplankton.") + } + zoop_list<-list() biomass_join<-function(d, level){ diff --git a/R/zoop_downloader.R b/R/zoop_downloader.R index b613c68..a7c6e56 100644 --- a/R/zoop_downloader.R +++ b/R/zoop_downloader.R @@ -8,11 +8,10 @@ #' "DOP" (Directed Outflow Project Lower Trophic Study), and "YBFMP" (Yolo Bypass Fish Monitoring Program). #' @param Data_sets Datasets to include in combined data. Choices include "EMP_Meso", "FMWT_Meso", "STN_Meso", "20mm_Meso", "FRP_Meso", "YBFMP_Meso", "EMP_Micro", "YBFMP_Micro", "FRP_Macro", "EMP_Macro", "FMWT_Macro", "STN_Macro", "DOP_Macro", and "DOP_Meso". Defaults to including all datasets except the two YBFMP datasets. #' @param Biomass Whether to add carbon biomass (carbon biomass per unit effort (\eqn{\mu}g/ \ifelse{html}{\out{m3}}{\eqn{m^{3}}})) to the dataset (where conversion equations and required data are available). Defaults to \code{Biomass = TRUE} -#' @param Data_folder Path to folder in which source datasets are stored, and to which you would like datasets to be downloaded if you set \code{Redownload_data = TRUE}. If you do not want to store every source dataset, you can leave this at the default \code{tempdir()}. If you do not wish to redownload these datasets every time you run the function, you can set this to a directory on your computer and run the function in the future with \code{Redownload_data = FALSE}, which will load the source datasets from \code{Data_folder} instead of downloading them again. +#' @param Data_folder Path to folder to which you would like datasets to be downloaded. If you do not want to store every source dataset, you can leave this at the default \code{tempdir()}. #' @param Save_object Should the combined data be saved to disk? Defaults to \code{Save_object = TRUE}. #' @param Return_object Should data be returned as an R object? If \code{TRUE}, the function will return the full combined dataset. Defaults to `Return_object = FALSE`. #' @param Return_object_type If \code{Return_object = TRUE}, should data be returned as a combined dataframe (\code{Return_object_type = "Combined"}) or a list with component "Zooplankton" containing the zooplankton data and component "Environment" containing the environmental data (\code{Return_object_type = "List"}, the default). A list is required to feed data into the \code{Zoopsynther} function without saving the combined dataset to disk. -#' @param Redownload_data Should source datasets be redownloaded from the internet? Defaults to \code{Redownload_data = FALSE}. #' @param Download_method Method used to download files. See argument \code{method} options in \code{\link[utils]{download.file}}. Defaults to "curl". #' @param Zoop_path File path specifying the folder and filename of the zooplankton dataset. Defaults to \code{Zoop_path = file.path(Data_folder, "zoopforzooper")}. #' @param Env_path File path specifying the folder and filename of the dataset with accessory environmental parameters. Defaults to \code{Env_path = file.path(Data_folder, "zoopenvforzooper")}. @@ -28,7 +27,7 @@ #' @examples #' \dontrun{ #' Data <- Zoopdownloader(Data_folder = tempdir(), Return_object = TRUE, -#' Save_object = FALSE, Redownload_data = TRUE) +#' Save_object = FALSE) #' } #' @seealso \code{\link{Zoopsynther}}, \code{\link{crosswalk}}, \code{\link{stations}}, \code{\link{zooper}} #' @export @@ -43,7 +42,6 @@ Zoopdownloader <- function( Save_object = TRUE, Return_object = FALSE, Return_object_type = "List", - Redownload_data = FALSE, Download_method="auto", Zoop_path = file.path(Data_folder, "zoopforzooper"), Env_path = file.path(Data_folder, "zoopenvforzooper"), @@ -69,12 +67,8 @@ Zoopdownloader <- function( stop("Return_object_type must be either 'List' or 'Combined'.") } - if(!purrr::every(list(Save_object, Return_object, Redownload_data), is.logical)){ - stop("Save_object, Return_object, and Redownload_data must all have logical arguments.") - } - - if(Biomass & !("Macro"%in%stringr::str_extract(Data_sets, "(?<=_).*") & "EMP_Macro"%in%Data_sets)){ - stop("Biomass are only available for macrozooplankton, and currently only available for EMP, so EMP_Macro must be selected if Length = TRUE.") + if(!purrr::every(list(Save_object, Return_object), is.logical)){ + stop("Save_object and Return_object must each have logical arguments.") } # Load station key to later incorporate latitudes and longitudes @@ -97,15 +91,11 @@ Zoopdownloader <- function( if("EMP_Meso"%in%Data_sets) { #download the file - if (!file.exists(file.path(Data_folder, "EMP_meso.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$EMP$Meso, - destfile=file.path(Data_folder, "EMP_meso.csv"), mode="wb", method=Download_method) - } + raw_zoo_EMP_Meso <- EDIutils::read_data_entity(URLs$EMP$PID, URLs$EMP$Meso) + # Import the EMP data - # Import the EMP data - - zoo_EMP_Meso<-readr::read_csv(file.path(Data_folder, "EMP_meso.csv"), + zoo_EMP_Meso<-readr::read_csv(raw_zoo_EMP_Meso, col_types=readr::cols_only(SampleDate="c", Time="c", StationNZ="c", Chl_a="d", Secchi="d", Temperature="d", ECSurfacePreTow="d", ECBottomPreTow="d", @@ -179,20 +169,13 @@ Zoopdownloader <- function( # DOP Meso --------------------------------------------------------------------- if("DOP_Meso"%in%Data_sets) { - #download the files - if (!file.exists(file.path(Data_folder, "DOP_Meso.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$DOP$Meso, - destfile=file.path(Data_folder, "DOP_Meso.csv"), mode="wb", method=Download_method) - } - if (!file.exists(file.path(Data_folder, "DOP_trawls.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$DOP$trawls, - destfile=file.path(Data_folder, "DOP_trawls.csv"), mode="wb", method=Download_method) - } - + #download the file + raw_zoo_DOP_Meso <- EDIutils::read_data_entity(URLs$DOP$PID, URLs$DOP$Meso) + raw_zoo_DOP_trawls <- EDIutils::read_data_entity(URLs$DOP$PID, URLs$DOP$trawls) # Import the DOP data - zoo_DOP_Meso<-readr::read_csv(file.path(Data_folder, "DOP_Meso.csv"), + zoo_DOP_Meso<-readr::read_csv(raw_zoo_DOP_Meso, col_types=readr::cols_only(ICF_ID="c", Acanthocyclops_spp_adult="d", Acanthocyclops_vernalis_adult="d", Acanthocyclops_vernalis_copepodid="d", Acartia_spp_adult="d", Acartia_spp_copepodid="d", Acartiella_sinensis_adult="d", Acartiella_sinensis_copepodid="d", Asplanchna_spp="d", @@ -221,7 +204,7 @@ Zoopdownloader <- function( Synchaeta_spp="d", Tortanus_dextrilobatus_adult="d", Tortanus_discaudatus_adult="d", Tortanus_spp_copepodid="d", Trichocerca_spp="d")) - zoo_DOP_trawls<-readr::read_csv(file.path(Data_folder, "DOP_trawls.csv"), + zoo_DOP_trawls<-readr::read_csv(raw_zoo_DOP_trawls, col_types=readr::cols_only(ICF_ID="c", Date="c", Start_Time="c", Station_Code="c", Habitat="c", Latitude="d", Longitude="d", Start_Depth="d", Temperature="d", Conductivity="d", @@ -279,20 +262,13 @@ Zoopdownloader <- function( # DOP Macro --------------------------------------------------------------------- if("DOP_Macro"%in%Data_sets) { - #download the files - if (!file.exists(file.path(Data_folder, "DOP_Macro.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$DOP$Macro, - destfile=file.path(Data_folder, "DOP_Macro.csv"), mode="wb", method=Download_method) - } - if (!file.exists(file.path(Data_folder, "DOP_trawls.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$DOP$trawls, - destfile=file.path(Data_folder, "DOP_trawls.csv"), mode="wb", method=Download_method) - } - + #download the file + raw_zoo_DOP_Macro <- EDIutils::read_data_entity(URLs$DOP$PID, URLs$DOP$Macro) + raw_zoo_DOP_trawls <- EDIutils::read_data_entity(URLs$DOP$PID, URLs$DOP$trawls) # Import the DOP data - zoo_DOP_Macro<-readr::read_csv(file.path(Data_folder, "DOP_Macro.csv"), + zoo_DOP_Macro<-readr::read_csv(raw_zoo_DOP_Macro, col_types=readr::cols_only(ICF_ID="c", Alienacanthomysis_macropsis="d", Americorophium_spinicorne="d", Americorophium_spp="d", Americorophium_stimpsoni="d", Ampelisca_abdita="d", Amphipod_UNID="d", Ampithoe_spp="d", Ampithoe_valida="d", @@ -305,7 +281,7 @@ Zoopdownloader <- function( Orientomysis_aspera="d", Orientomysis_hwanhaiensis="d", Pleustidae_UNID="d", Shrimp_UNID_larvae="d", Sinocorophium_alienense="d", Tanaidacea_UNID="d")) - zoo_DOP_trawls<-readr::read_csv(file.path(Data_folder, "DOP_trawls.csv"), + zoo_DOP_trawls<-readr::read_csv(raw_zoo_DOP_trawls, col_types=readr::cols_only(ICF_ID="c", Date="c", Start_Time="c", Station_Code="c", Habitat="c", Latitude="d", Longitude="d", Start_Depth="d", Temperature="d", Conductivity="d", @@ -365,20 +341,15 @@ Zoopdownloader <- function( if("FMWT_Meso"%in%Data_sets | "STN_Meso"%in%Data_sets) { #download the file - if (!file.exists(file.path(Data_folder, "FMWTSTN_Meso.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$FMWTSTN$Meso, - destfile=file.path(Data_folder,"FMWTSTN_Meso.csv"), mode="wb", method=Download_method) - } + raw_zoo_FMWT_Meso <- EDIutils::read_data_entity(URLs$FMWTSTN$PID, URLs$FMWTSTN$Meso) - if (!file.exists(file.path(Data_folder, "SMSCG_Meso.csv")) | Redownload_data) { Tryer(n=3, fun=utils::download.file, url=URLs$SMSCG$Meso, destfile=file.path(Data_folder, "SMSCG_Meso.csv"), mode="wb", method=Download_method) - } # Import the FMWT data - zoo_FMWT_Meso <- readr::read_csv(file.path(Data_folder, "FMWTSTN_Meso.csv"), + zoo_FMWT_Meso <- readr::read_csv(raw_zoo_FMWT_Meso, col_types=readr::cols_only(Project="c", Year="d", Survey="d", Date="c", Station="c", Time="c", TideCode="c", DepthBottom="d", CondSurf="d", @@ -495,10 +466,8 @@ Zoopdownloader <- function( #download the file - if (!file.exists(file.path(Data_folder, "twentymm_Meso.csv")) | Redownload_data) { Tryer(n=3, fun=utils::download.file, url=URLs$twentymm$Meso, destfile=file.path(Data_folder, "twentymm_Meso.csv"), mode="wb", method=Download_method) - } @@ -568,16 +537,11 @@ Zoopdownloader <- function( # Import the FRP data #download the file - if (!file.exists(file.path(Data_folder, "zoopsFRP.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$FRP$Meso, - destfile=file.path(Data_folder, "zoopsFRP.csv"), mode="wb", method=Download_method) - Tryer(n=3, fun=utils::download.file, url=URLs$FRP$site, - destfile=file.path(Data_folder, "sitesFRP.csv"), mode="wb", method=Download_method) - - } + raw_zoo_FRP_Meso <- EDIutils::read_data_entity(URLs$FRP$PID, URLs$FRP$Meso) + raw_sites_FRP_Meso <- EDIutils::read_data_entity(URLs$FRP$PID, URLs$FRP$site) - zoo_FRP_Meso <- readr::read_csv(file.path(Data_folder, "zoopsFRP.csv"), na=c("", "NA")) - sites_FRP_Meso <- readr::read_csv(file.path(Data_folder, "sitesFRP.csv"), na=c("", "NA")) + zoo_FRP_Meso <- readr::read_csv(raw_zoo_FRP_Meso, na=c("", "NA")) + sites_FRP_Meso <- readr::read_csv(raw_sites_FRP_Meso, na=c("", "NA")) #join environmental data to taxa counts and fix some wonky names FRP_all = dplyr::left_join(zoo_FRP_Meso, sites_FRP_Meso) %>% @@ -641,12 +605,9 @@ Zoopdownloader <- function( if("YBFMP_Meso"%in%Data_sets | "YBFMP_Micro"%in%Data_sets) { #download the file - if (!file.exists(file.path(Data_folder, "YBFMP.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$YBFMP, - destfile=file.path(Data_folder, "YBFMP.csv"), mode="wb", method=Download_method) - } + raw_zoo_YBFMP <- EDIutils::read_data_entity(URLs$YBFMP$PID, URLs$YBFMP$Meso) - zoo_YBFMP<-readr::read_csv(file.path(Data_folder, "YBFMP.csv"), + zoo_YBFMP<-readr::read_csv(raw_zoo_YBFMP, col_types = readr::cols_only(Date="c", Time="c", StationCode="c", Tide="c", WaterTemperature="d", Secchi="d", SpCnd="d", pH="d", DO="d", Turbidity="d", @@ -731,13 +692,10 @@ Zoopdownloader <- function( if("EMP_Micro"%in%Data_sets) { #download the file - if (!file.exists(file.path(Data_folder, "EMP_Micro.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$EMP$Micro, - destfile=file.path(Data_folder, "EMP_Micro.csv"), mode="wb", method=Download_method) - } + raw_zoo_EMP_Micro <- EDIutils::read_data_entity(URLs$EMP$PID, URLs$EMP$Micro) # Import the EMP data - zoo_EMP_Micro<-readr::read_csv(file.path(Data_folder, "EMP_Micro.csv"), + zoo_EMP_Micro<-readr::read_csv(raw_zoo_EMP_Micro, col_types=readr::cols_only(SampleDate="c", StationNZ="c", Chl_a="d", Secchi="d", Temperature="d", ECSurfacePreTow="d", ECBottomPreTow="d", @@ -801,16 +759,11 @@ Zoopdownloader <- function( if("FRP_Macro"%in%Data_sets) { #download the file - if (!file.exists(file.path(Data_folder, "macroinvert_FRP.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$FRP$Macro, - destfile=file.path(Data_folder, "macroinvert_FRP.csv"), mode="wb", method=Download_method) - Tryer(n=3, fun=utils::download.file, url=URLs$FRP$site, - destfile=file.path(Data_folder, "sitesFRP.csv"), mode="wb", method=Download_method) + raw_zoo_FRP_Macro <- EDIutils::read_data_entity(URLs$FRP$PID, URLs$FRP$Macro) + raw_sites_FRP_Macro <- EDIutils::read_data_entity(URLs$FRP$PID, URLs$FRP$site) - } - - zoo_FRP_Macro <- readr::read_csv(file.path(Data_folder, "macroinvert_FRP.csv"), na=c("", "NA")) - sites_FRP_Macro <- readr::read_csv(file.path(Data_folder, "sitesFRP.csv"), na=c("", "NA")) + zoo_FRP_Macro <- readr::read_csv(raw_zoo_FRP_Macro, na=c("", "NA")) + sites_FRP_Macro <- readr::read_csv(raw_sites_FRP_Macro, na=c("", "NA")) #join environmental data to taxa counts and fix some wonky names FRP_allmac = dplyr::left_join(dplyr::select(zoo_FRP_Macro, -"Date", -"Location"), sites_FRP_Macro, by = "VisitNo") %>% @@ -878,14 +831,11 @@ Zoopdownloader <- function( if("EMP_Macro"%in%Data_sets) { #download the file - if (!file.exists(file.path(Data_folder, "EMP_Macro.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$EMP$Macro, - destfile=file.path(Data_folder, "EMP_Macro.csv"), mode="wb", method=Download_method) - } + raw_zoo_EMP_Macro <- EDIutils::read_data_entity(URLs$EMP$PID, URLs$EMP$Macro) # Import the EMP data - zoo_EMP_Macro<-readr::read_csv(file.path(Data_folder, "EMP_Macro.csv"), + zoo_EMP_Macro<-readr::read_csv(raw_zoo_EMP_Macro, col_types=readr::cols_only(SampleDate="c", Time="c", StationNZ="c", Chl_a="d", Secchi="d", Temperature="d", ECSurfacePreTow="d", ECBottomPreTow="d", @@ -941,12 +891,11 @@ Zoopdownloader <- function( cat("\nEMP_Macro finished!\n\n") if(Biomass){ + #download the file - if (!file.exists(file.path(Data_folder, "EMP_Lengths.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$EMP$Lengths, - destfile=file.path(Data_folder, "EMP_Lengths.csv"), mode="wb", method=Download_method) + raw_zoo_EMP_lengths <- EDIutils::read_data_entity(URLs$EMP$PID, URLs$EMP$Lengths) - lengths.list[["EMP_Lengths"]]<-readr::read_csv(file.path(Data_folder, "EMP_Lengths.csv"), + lengths.list[["EMP_Lengths"]]<-readr::read_csv(raw_zoo_EMP_lengths, col_types=readr::cols_only(SampleDate="c", StationNZ="c", SpeciesName="c", Length="d", AdjustedFreq="d"))%>% dplyr::mutate(SampleDate=lubridate::parse_date_time(.data$SampleDate, "%m/%d/%Y", tz="America/Los_Angeles"))%>% @@ -964,7 +913,7 @@ Zoopdownloader <- function( dplyr::select(-"EMP_Lengths", -"Date", -"Station") cat("\nEMP_Macro lengths finished!\n\n") - } + } } @@ -973,19 +922,14 @@ Zoopdownloader <- function( if("FMWT_Macro"%in%Data_sets | "STN_Macro"%in%Data_sets) { #download the file - if (!file.exists(file.path(Data_folder, "FMWTSTN_Macro.csv")) | Redownload_data) { - Tryer(n=3, fun=utils::download.file, url=URLs$FMWTSTN$Macro, - destfile=file.path(Data_folder,"FMWTSTN_Macro.csv"), mode="wb", method=Download_method) - } + raw_zoo_FMWT_Macro <- EDIutils::read_data_entity(URLs$FMWTSTN$PID, URLs$FMWTSTN$Macro) #download the file - if (!file.exists(file.path(Data_folder, "SMSCG_Macro.csv")) | Redownload_data) { Tryer(n=3, fun=utils::download.file, url=URLs$SMSCG$Macro, destfile=file.path(Data_folder, "SMSCG_Macro.csv"), mode="wb", method=Download_method) - } - zoo_FMWT_Macro <- readr::read_csv(file.path(Data_folder, "FMWTSTN_Macro.csv"), + zoo_FMWT_Macro <- readr::read_csv(raw_zoo_FMWT_Macro, col_types=readr::cols_only(Project="c", Year="d", Survey="d", Date="c", Station="c", Time="c", TideCode="c", DepthBottom="d", CondSurf="d", @@ -1023,7 +967,7 @@ Zoopdownloader <- function( #put date and time in the right format. Date first, tehn datetime to try and fix a problem where the date randomly came out wrong in the 'datetime' version dplyr::mutate(Date=lubridate::parse_date_time(.data$Date, "%m/%d/%y", tz="America/Los_Angeles"), Datetime = lubridate::parse_date_time(dplyr::if_else(is.na(.data$Time), NA_character_, - paste(.data$Date, .data$Time)), "%Y-%m-%d %H:%M:%S", tz="America/Los_Angeles")) + paste(.data$Date, .data$Time)), orders="%Y-%m-%d %H:%M", tz="America/Los_Angeles")) data.list[["FMWT_Macro"]] <- dplyr::bind_rows(zoo_FMWT_Macro, zoo_SMSCG_Macro)%>% dplyr::select(-"ID")%>% diff --git a/R/zoop_synthesizer.R b/R/zoop_synthesizer.R index 5dbcf17..249f371 100644 --- a/R/zoop_synthesizer.R +++ b/R/zoop_synthesizer.R @@ -20,8 +20,7 @@ #' @param Temp_range Same as \code{Sal_bott_range} but for surface temperature. #' @param Lat_range Latitude range to include in the final dataset. Include a vector of length 2 specifying the minimum and maximum values you wish to include, in decimal degree format. Defaults to \code{Lat_range = NA}, which includes all latitudes. #' @param Long_range Same as previous, but for longitude. Don't forget that Longitudes should be negative in the Delta! -#' @param Reload_data If set to \code{Reload_data = T} runs the \code{\link{Zoopdownloader}} function to re-combine source datasets. To include local versions of the datasets without redownloading them from online, set \code{Reload_data = TRUE} and \code{Redownload_data = FALSE}. Defaults to \code{Reload_data= FALSE} -#' @param Redownload_data Should data be re-downloaded from the internet? If set to \code{Redownload_data = TRUE}, runs \code{\link{Zoopdownloader}(Redownload_data=Redownload_data, Zoop_path=Zoop_path, Env_path=Env_path, ...)}. Defaults to \code{Redownload_data = FALSE}. +#' @param Redownload_data Should data be re-downloaded from the internet? If set to \code{Redownload_data = TRUE}, redownloads the data using \code{\link{Zoopdownloader}}. Defaults to \code{Redownload_data = FALSE}. #' @param All_env Should all environmental parameters be included? Defaults to \code{All_env = TRUE}. #' @param Shiny Is this function being used within the shiny app? If set to \code{Shiny = TRUE}, outputs a list with the integrated dataset as one component and the caveats as the other component. Defaults to \code{Shiny = FALSE}. #' @inheritParams Zoopdownloader @@ -85,7 +84,6 @@ Zoopsynther<-function( Temp_range = NA, Lat_range = NA, Long_range = NA, - Reload_data = F, Redownload_data = F, All_env = T, Shiny = F, @@ -132,8 +130,8 @@ Zoopsynther<-function( stop("Response must contain one or more of the following options: CPUE, BPUE") } - if(!purrr::every(list(Shiny, Reload_data, Redownload_data, All_env, Time_consistency), is.logical)){ - stop("Shiny, All_env, Reload_data, Redownload_data, and Time_consistency must all have logical arguments.") + if(!purrr::every(list(Shiny, Redownload_data, All_env, Time_consistency), is.logical)){ + stop("Shiny, All_env, Redownload_data, and Time_consistency must all have logical arguments.") } if(Data_type=="Taxa" & Time_consistency){ @@ -153,8 +151,14 @@ Zoopsynther<-function( } #Make it possible to re-download data if desired - if(Reload_data | Redownload_data){ - Zoopdownloader(Redownload_data = Redownload_data, Zoop_path = Zoop_path, Env_path = Env_path, ...) + if(Redownload_data){ + data_sets<-paste(rep(Sources, times=length(Size_class)), rep(Size_class, each=length(Sources)), sep="_") + data_sets<-data_sets[which(data_sets%in%c("EMP_Meso", "FMWT_Meso", "STN_Meso", + "20mm_Meso", "FRP_Meso", "EMP_Micro", + "FRP_Macro", "EMP_Macro", "FMWT_Macro", + "STN_Macro", "DOP_Meso", "DOP_Macro"))] + + Zoopdownloader(Data_sets=data_sets, Zoop_path = Zoop_path, Env_path = Env_path, ...) } #Recode Source diff --git a/data-raw/crosswalk-zoopComb-zoopEnvComb-startDates.R b/data-raw/crosswalk-zoopComb-zoopEnvComb-startDates.R index f5515bf..cbcbf9f 100644 --- a/data-raw/crosswalk-zoopComb-zoopEnvComb-startDates.R +++ b/data-raw/crosswalk-zoopComb-zoopEnvComb-startDates.R @@ -29,7 +29,7 @@ zoop<-Zoopdownloader(Data_sets=c("EMP_Meso", "FMWT_Meso", "STN_Meso", "FRP_Macro", "EMP_Macro", "FMWT_Macro", "STN_Macro", "YBFMP_Meso", "YBFMP_Micro", "DOP_Macro", "DOP_Meso"), Data_folder=tempdir(), Save_object=FALSE, Return_object=TRUE, - Redownload_data=TRUE, Crosswalk=crosswalk, Biomass=TRUE) + Crosswalk=crosswalk, Biomass=TRUE) zoopComb <- zoop$Zooplankton diff --git a/man/Zoopbiomass.Rd b/man/Zoopbiomass.Rd index 64f4628..f93238f 100644 --- a/man/Zoopbiomass.Rd +++ b/man/Zoopbiomass.Rd @@ -7,7 +7,7 @@ This function converts zooplankton CPUE to carbon biomass (Carbon biomass per un \usage{ Zoopbiomass( Zoop, - ZoopLengths, + ZoopLengths = NULL, Biomass_mesomicro = zooper::biomass_mesomicro, Biomass_macro = zooper::biomass_macro ) @@ -15,7 +15,7 @@ Zoopbiomass( \arguments{ \item{Zoop}{Zooplankton count dataset} -\item{ZoopLengths}{Zooplankton length dataset for macrozooplankton.} +\item{ZoopLengths}{Zooplankton length dataset for macrozooplankton. May be left NULL if the Macro size class is not included.} \item{Biomass_mesomicro}{The micro and meso zooplankton biomass conversion table. The default is \code{\link{biomass_mesomicro}}} diff --git a/man/Zoopdownloader.Rd b/man/Zoopdownloader.Rd index 2ecc856..4e942e3 100644 --- a/man/Zoopdownloader.Rd +++ b/man/Zoopdownloader.Rd @@ -13,7 +13,6 @@ Zoopdownloader( Save_object = TRUE, Return_object = FALSE, Return_object_type = "List", - Redownload_data = FALSE, Download_method = "auto", Zoop_path = file.path(Data_folder, "zoopforzooper"), Env_path = file.path(Data_folder, "zoopenvforzooper"), @@ -26,7 +25,7 @@ Zoopdownloader( \item{Biomass}{Whether to add carbon biomass (carbon biomass per unit effort (\eqn{\mu}g/ \ifelse{html}{\out{m3}}{\eqn{m^{3}}})) to the dataset (where conversion equations and required data are available). Defaults to \code{Biomass = TRUE}} -\item{Data_folder}{Path to folder in which source datasets are stored, and to which you would like datasets to be downloaded if you set \code{Redownload_data = TRUE}. If you do not want to store every source dataset, you can leave this at the default \code{tempdir()}. If you do not wish to redownload these datasets every time you run the function, you can set this to a directory on your computer and run the function in the future with \code{Redownload_data = FALSE}, which will load the source datasets from \code{Data_folder} instead of downloading them again.} +\item{Data_folder}{Path to folder to which you would like datasets to be downloaded. If you do not want to store every source dataset, you can leave this at the default \code{tempdir()}.} \item{Save_object}{Should the combined data be saved to disk? Defaults to \code{Save_object = TRUE}.} @@ -34,8 +33,6 @@ Zoopdownloader( \item{Return_object_type}{If \code{Return_object = TRUE}, should data be returned as a combined dataframe (\code{Return_object_type = "Combined"}) or a list with component "Zooplankton" containing the zooplankton data and component "Environment" containing the environmental data (\code{Return_object_type = "List"}, the default). A list is required to feed data into the \code{Zoopsynther} function without saving the combined dataset to disk.} -\item{Redownload_data}{Should source datasets be redownloaded from the internet? Defaults to \code{Redownload_data = FALSE}.} - \item{Download_method}{Method used to download files. See argument \code{method} options in \code{\link[utils]{download.file}}. Defaults to "curl".} \item{Zoop_path}{File path specifying the folder and filename of the zooplankton dataset. Defaults to \code{Zoop_path = file.path(Data_folder, "zoopforzooper")}.} @@ -63,7 +60,7 @@ Note that EMP Macro samples with QAQC flags (any value of AmphipodCode other tha \examples{ \dontrun{ Data <- Zoopdownloader(Data_folder = tempdir(), Return_object = TRUE, -Save_object = FALSE, Redownload_data = TRUE) +Save_object = FALSE) } } \seealso{ diff --git a/man/Zoopsynther.Rd b/man/Zoopsynther.Rd index 9c67fb7..31fc711 100644 --- a/man/Zoopsynther.Rd +++ b/man/Zoopsynther.Rd @@ -24,7 +24,6 @@ Zoopsynther( Temp_range = NA, Lat_range = NA, Long_range = NA, - Reload_data = F, Redownload_data = F, All_env = T, Shiny = F, @@ -74,9 +73,7 @@ Zoopsynther( \item{Long_range}{Same as previous, but for longitude. Don't forget that Longitudes should be negative in the Delta!} -\item{Reload_data}{If set to \code{Reload_data = T} runs the \code{\link{Zoopdownloader}} function to re-combine source datasets. To include local versions of the datasets without redownloading them from online, set \code{Reload_data = TRUE} and \code{Redownload_data = FALSE}. Defaults to \code{Reload_data= FALSE}} - -\item{Redownload_data}{Should data be re-downloaded from the internet? If set to \code{Redownload_data = TRUE}, runs \code{\link{Zoopdownloader}(Redownload_data=Redownload_data, Zoop_path=Zoop_path, Env_path=Env_path, ...)}. Defaults to \code{Redownload_data = FALSE}.} +\item{Redownload_data}{Should data be re-downloaded from the internet? If set to \code{Redownload_data = TRUE}, redownloads the data using \code{\link{Zoopdownloader}}. Defaults to \code{Redownload_data = FALSE}.} \item{All_env}{Should all environmental parameters be included? Defaults to \code{All_env = TRUE}.} diff --git a/man/crosswalk.Rd b/man/crosswalk.Rd index 8a259ec..5a0d52b 100644 --- a/man/crosswalk.Rd +++ b/man/crosswalk.Rd @@ -5,7 +5,7 @@ \alias{crosswalk} \title{Taxonomic crosswalk among datasets} \format{ -a tibble with 404 rows and 34 variables +a tibble with 471 rows and 37 variables \describe{ \item{EMP_Micro}{Taxonomic codes used in the Environmental Monitoring Program microzooplankton (43 \eqn{\mu}m) mesh dataset} \item{EMP_Meso}{Taxonomic codes used in the Environmental Monitoring Program mesozooplankton (160 \eqn{\mu}m) mesh dataset} @@ -39,6 +39,8 @@ a tibble with 404 rows and 34 variables \item{twentymmstart2}{First year the 20mm Survey restarted counting this taxa} \item{DOPstart}{First year DOP starting counting this taxa} \item{DOPend}{Last year DOP counted this taxa} + \item{FRPstart}{First year FRP starting counting this taxa} + \item{FRPend}{Last year FRP counted this taxa} } } \usage{ diff --git a/man/edi_entity_list.Rd b/man/edi_entity_list.Rd new file mode 100644 index 0000000..2a4005c --- /dev/null +++ b/man/edi_entity_list.Rd @@ -0,0 +1,15 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/Download_utilities.R +\name{edi_entity_list} +\alias{edi_entity_list} +\title{Create a list of entities from an EDI package} +\usage{ +edi_entity_list(PID) +} +\arguments{ +\item{PID}{EDI package ID} +} +\description{ +This function lists all data entities for an EDI package +} +\keyword{internal} diff --git a/man/stations.Rd b/man/stations.Rd index 84d81a2..b2d7196 100644 --- a/man/stations.Rd +++ b/man/stations.Rd @@ -5,7 +5,7 @@ \alias{stations} \title{Station locations} \format{ -a tibble with 387 rows and 4 columns +a tibble with 399 rows and 4 columns \describe{ \item{Source}{Abbreviated name of the source dataset} \item{Station}{Sampling station name} diff --git a/man/stationsEMPEZ.Rd b/man/stationsEMPEZ.Rd index 7dd0a70..8a7c402 100644 --- a/man/stationsEMPEZ.Rd +++ b/man/stationsEMPEZ.Rd @@ -5,7 +5,7 @@ \alias{stationsEMPEZ} \title{EMP EZ Station locations} \format{ -a tibble with 491 rows and 4 columns +a tibble with 575 rows and 4 columns \describe{ \item{Date}{Date sample was collected} \item{Station}{Sampling station name} diff --git a/man/zoopComb.Rd b/man/zoopComb.Rd index 681e613..0876e97 100644 --- a/man/zoopComb.Rd +++ b/man/zoopComb.Rd @@ -5,7 +5,7 @@ \alias{zoopComb} \title{Combined zooplankton dataset} \format{ -a tibble with 3,615,105 rows and 14 columns. +a tibble with 3,833,994 rows and 14 columns. \describe{ \item{Source}{Abbreviated name of the source dataset. "EMP"=Environmental Monitoring Program, "FRP"=Fish Restoration Program, "FMWT"= Fall Midwater Trawl, "STN"= Townet Survey, "20mm" =20mm survey, "DOP" = Directed Outflow Project Lower Trophic Study, and "YBFMP"= Yolo Bypass Fish Monitoring Program.} \item{SizeClass}{Net size class. Micro corresponds to 43-50 \eqn{\mu}m mesh, Meso corresponds to 150-160 \eqn{\mu}m mesh, and Macro corresponds to 500-505 \eqn{\mu}m mesh. However, prior to 1974 EMP macrozooplankton were sampled with a 930 \eqn{\mu}m mesh net.} diff --git a/man/zoopEnvComb.Rd b/man/zoopEnvComb.Rd index ca7cb17..e9150d4 100644 --- a/man/zoopEnvComb.Rd +++ b/man/zoopEnvComb.Rd @@ -6,7 +6,7 @@ \alias{zoopEnvComb} \title{Environmental data} \format{ -a tibble with 44,690 rows and 20 columns +a tibble with 48,933 rows and 20 columns \describe{ \item{Source}{Abbreviated name of the source dataset. "EMP"=Environmental Monitoring Program, "FRP"=Fish Restoration Program, "FMWT"= Fall Midwater Trawl, "STN"= Townet Survey, "20mm" =20mm survey, "DOP" = Directed Outflow Project Lower Trophic Study, and "YBFMP"= Yolo Bypass Fish Monitoring Program.} \item{Year}{Year sample was collected} diff --git a/man/zooper.Rd b/man/zooper.Rd index 076da0f..442c48a 100644 --- a/man/zooper.Rd +++ b/man/zooper.Rd @@ -99,6 +99,7 @@ Useful links: Authors: \itemize{ + \item Samuel M Bashevkin \email{sam.bashevkin@waterboards.ca.gov} (\href{https://orcid.org/0000-0001-7406-7089}{ORCID}) \item Rosemary Hartman \email{Rosemary.Hartman@water.ca.gov} (\href{https://orcid.org/0000-0002-0642-183X}{ORCID}) \item Karrin Alstad \email{Karrin.Alstad@wildlife.ca.gov} \item Catarina Pien \email{cpien@usbr.gov} (\href{https://orcid.org/0000-0003-4427-6300}{ORCID}) diff --git a/tests/testthat/test-Source_datasets.R b/tests/testthat/test-Source_datasets.R index 6f05b3f..fa8b056 100644 --- a/tests/testthat/test-Source_datasets.R +++ b/tests/testthat/test-Source_datasets.R @@ -15,22 +15,21 @@ Data_folder<-tempdir() # EMP Meso ---------------------------------------------------------------- -Tryer(n=3, fun=download.file, url=URLs$EMP$Meso, destfile=file.path(Data_folder, "EMP_Meso.csv"), mode="wb", method=Download_method) +raw_zoo_EMP_Meso <- EDIutils::read_data_entity(URLs$EMP$PID, URLs$EMP$Meso) -names_EMP_Meso<-readr::read_csv(file.path(Data_folder, "EMP_Meso.csv"), col_types = cols(.default=col_character()))%>% +names_EMP_Meso<-readr::read_csv(raw_zoo_EMP_Meso, col_types = cols(.default=col_character()))%>% names() # FMWT STN Meso ----------------------------------------------------------- -Tryer(n=3, fun=download.file, url=URLs$FMWTSTN$Meso, - destfile=file.path(Data_folder, "FMWTSTN_Meso.csv"), mode="wb", method=Download_method) +raw_zoo_FMWT_Meso <- EDIutils::read_data_entity(URLs$FMWTSTN$PID, URLs$FMWTSTN$Meso) Tryer(n=3, fun=download.file, url=URLs$SMSCG$Meso, destfile=file.path(Data_folder, "SMSCG_Meso.csv"), mode="wb", method=Download_method) -names_FMWTSTN_Meso<-readr::read_csv(file.path(Data_folder, "FMWTSTN_Meso.csv"), +names_FMWTSTN_Meso<-readr::read_csv(raw_zoo_FMWT_Meso, col_types = "c")%>% names() @@ -52,70 +51,63 @@ names_20mm_Meso<-readxl::read_excel(file.path(Data_folder, "twentymm_Meso.csv"), # FRP ---------------------------------------------------------------- -Tryer(n=3, fun=utils::download.file, url=URLs$FRP$Meso, - destfile=file.path(Data_folder, "zoopsFRP.csv"), mode="wb", method=Download_method) -Tryer(n=3, fun=utils::download.file, url=URLs$FRP$Macro, - destfile=file.path(Data_folder, "macroinvert_FRP.csv"), mode="wb", method=Download_method) -Tryer(n=3, fun=utils::download.file, url=URLs$FRP$site, - destfile=file.path(Data_folder, "sitesFRP.csv"), mode="wb", method=Download_method) +raw_zoo_FRP_Meso <- EDIutils::read_data_entity(URLs$FRP$PID, URLs$FRP$Meso) +raw_sites_FRP <- EDIutils::read_data_entity(URLs$FRP$PID, URLs$FRP$site) +raw_zoo_FRP_Macro <- EDIutils::read_data_entity(URLs$FRP$PID, URLs$FRP$Macro) -names_FRP_Meso <- readr::read_csv(file.path(Data_folder, "zoopsFRP.csv"), na=c("", "NA"))%>% +names_FRP_Meso <- readr::read_csv(raw_zoo_FRP_Meso, na=c("", "NA"))%>% names() -names_FRP_Macro <- readr::read_csv(file.path(Data_folder, "macroinvert_FRP.csv"), na=c("", "NA"))%>% +names_FRP_Macro <- readr::read_csv(raw_zoo_FRP_Macro, na=c("", "NA"))%>% names() -names_FRP_sites <- readr::read_csv(file.path(Data_folder, "sitesFRP.csv"), na=c("", "NA"))%>% +names_FRP_sites <- readr::read_csv(raw_sites_FRP, na=c("", "NA"))%>% names() # YBFMP Meso/Micro -------------------------------------------------------- -Tryer(n=3, fun=utils::download.file, url=URLs$YBFMP, - destfile=file.path(Data_folder, "YBFMP.csv"), mode="wb", method=Download_method) +raw_zoo_YBFMP <- EDIutils::read_data_entity(URLs$YBFMP$PID, URLs$YBFMP$Meso) -names_YBFMP<-readr::read_csv(file.path(Data_folder, "YBFMP.csv"), +names_YBFMP<-readr::read_csv(raw_zoo_YBFMP, col_types = cols(.default=col_character()))%>% names() # EMP Micro --------------------------------------------------------------- -Tryer(n=3, fun=download.file, url=URLs$EMP$Micro, - destfile=file.path(Data_folder, "EMP_Micro.csv"), mode="wb", method=Download_method) +raw_zoo_EMP_Micro <- EDIutils::read_data_entity(URLs$EMP$PID, URLs$EMP$Micro) -names_EMP_Micro<-readr::read_csv(file.path(Data_folder, "EMP_Micro.csv"), +names_EMP_Micro<-readr::read_csv(raw_zoo_EMP_Micro, col_types=cols(.default=col_character()))%>% names() # EMP Macro --------------------------------------------------------------- -Tryer(n=3, fun=download.file, url=URLs$EMP$Macro, - destfile=file.path(Data_folder, "EMP_Macro.csv"), mode="wb", method=Download_method) +raw_zoo_EMP_Macro <- EDIutils::read_data_entity(URLs$EMP$PID, URLs$EMP$Macro) -names_EMP_Macro<-readr::read_csv(file.path(Data_folder, "EMP_Macro.csv"), +names_EMP_Macro<-readr::read_csv(raw_zoo_EMP_Macro, col_types=cols(.default=col_character()))%>% names() # EMP Lengths --------------------------------------------------------------- -Tryer(n=3, fun=download.file, url=URLs$EMP$Lengths, - destfile=file.path(Data_folder, "EMP_Lengths.csv"), mode="wb", method=Download_method) +raw_zoo_EMP_Lengths <- EDIutils::read_data_entity(URLs$EMP$PID, URLs$EMP$Lengths) -names_EMP_Lengths<-readr::read_csv(file.path(Data_folder, "EMP_Lengths.csv"), +names_EMP_Lengths<-readr::read_csv(raw_zoo_EMP_Lengths, col_types=cols(.default=col_character()))%>% select(-where(~all(is.na(.x))))%>% # drop empty columns names() # FMWT STN Macro ---------------------------------------------------------- -Tryer(n=3, fun=download.file, url=URLs$FMWTSTN$Macro, - destfile=file.path(Data_folder, "FMWTSTN_Macro.csv"), mode="wb", method=Download_method) +raw_zoo_FMWTSTN_Macro <- EDIutils::read_data_entity(URLs$FMWTSTN$PID, URLs$FMWTSTN$Macro) + Tryer(n=3, fun=download.file, url=URLs$SMSCG$Macro, destfile=file.path(Data_folder, "SMSCG_Macro.csv"), mode="wb", method=Download_method) -names_FMWT_Macro <- readr::read_csv(file.path(Data_folder, "FMWTSTN_Macro.csv"), +names_FMWT_Macro <- readr::read_csv(raw_zoo_FMWTSTN_Macro, col_types = cols(.default=col_character()))%>% names() @@ -127,28 +119,22 @@ names_SMSCG_Macro <- readr::read_csv(file.path(Data_folder, "SMSCG_Macro.csv"), # DOP Meso and Macro ------------------------------------------------------------------- #download the files -Tryer(n=3, fun=utils::download.file, url=URLs$DOP$Meso, - destfile=file.path(Data_folder, "DOP_Meso.csv"), mode="wb", method= Download_method) - -Tryer(n=3, fun=utils::download.file, url=URLs$DOP$trawls, - destfile=file.path(Data_folder, "DOP_trawls.csv"), mode="wb", method=Download_method) - -Tryer(n=3, fun=utils::download.file, url=URLs$DOP$Macro, - destfile=file.path(Data_folder, "DOP_Macro.csv"), mode="wb", method=Download_method) +raw_zoo_DOP_Meso <- EDIutils::read_data_entity(URLs$DOP$PID, URLs$DOP$Meso) +raw_zoo_DOP_Macro <- EDIutils::read_data_entity(URLs$DOP$PID, URLs$DOP$Macro) +raw_zoo_DOP_trawls <- EDIutils::read_data_entity(URLs$DOP$PID, URLs$DOP$trawls) -names_DOP_Meso<-readr::read_csv(file.path(Data_folder, "DOP_Meso.csv")) %>% +names_DOP_Meso<-readr::read_csv(raw_zoo_DOP_Meso) %>% names() -names_DOP_trawls<-readr::read_csv(file.path(Data_folder, "DOP_trawls.csv")) %>% + +names_DOP_trawls<-readr::read_csv(raw_zoo_DOP_trawls) %>% names() -names_DOP_Macro<-readr::read_csv(file.path(Data_folder, "DOP_Macro.csv")) %>% +names_DOP_Macro<-readr::read_csv(raw_zoo_DOP_Macro) %>% names() # Tests ------------------------------------------------------------------- - - test_that("EMP Meso column names have not changed", { expect_setequal(names_EMP_Meso, c('SurveyCode', 'Year', 'Survey', 'SurveyRep', 'SampleDate', 'StationNZ', 'EZStation', 'DWRStationNo', 'Core', 'Time', 'CDFW_Region', 'DWR_Region', 'Secchi', 'Chl_a', 'Temperature', 'ECSurfacePreTow', 'ECBottomPreTow', diff --git a/tests/testthat/test-Zoopdownloader.R b/tests/testthat/test-Zoopdownloader.R index 33f2e93..8d0cc2e 100644 --- a/tests/testthat/test-Zoopdownloader.R +++ b/tests/testthat/test-Zoopdownloader.R @@ -1,4 +1,5 @@ library(zooper) +library(withr) Data_sets <- c("EMP_Meso", "FMWT_Meso", "STN_Meso", "20mm_Meso", "FRP_Meso","EMP_Micro", @@ -7,7 +8,7 @@ Data_sets <- c("EMP_Meso", "FMWT_Meso", "STN_Meso", "DOP_Meso", "DOP_Macro") Data <- Zoopdownloader(Data_folder = tempdir(), Return_object = TRUE, - Save_object = FALSE, Redownload_data = TRUE, + Save_object = FALSE, Biomass=TRUE, Data_sets = Data_sets) No_coords2<-dplyr::filter(Data$Environment, is.na(Latitude) & !stringr::str_detect(Station, "NZEZ"))%>% @@ -20,6 +21,13 @@ No_stations<-dplyr::filter(Data$Environment, is.na(Station))%>% dplyr::mutate(ID=paste(Source, Date))%>% dplyr::pull(ID) +test_that("check_EDI_cred is working correctly", { + expect_error(with_envvar(new=c("EDI_API_KEY"="", "EDI_TOKEN"=""), check_EDI_cred()), + regexp="^An EDI API key is now required by EDI.*") + expect_no_error(with_envvar(new=c("EDI_API_KEY"="x"), check_EDI_cred())) + expect_no_error(with_envvar(new=c("EDI_TOKEN"="x"), check_EDI_cred())) +}) + test_that("Dowloaded data includes all datasets", { expect_setequal(unique(paste(Data$Zooplankton$Source, Data$Zooplankton$SizeClass, sep="_")), Data_sets) }) @@ -70,20 +78,20 @@ test_that("TowType only has the expected levels", { # BPUE test_that("When biomass is 0, CPUE is 0", { - expect_equal(unique(filter(Data$Zooplankton, SizeClass=="Macro" & BPUE==0)$CPUE), 0) + expect_equal(unique(dplyr::filter(Data$Zooplankton, SizeClass=="Macro" & BPUE==0)$CPUE), 0) }) test_that("When CPUE is 0, Biomass is 0", { - expect_equal(unique(filter(Data$Zooplankton, SizeClass=="Macro" & CPUE==0 & + expect_equal(unique(dplyr::filter(Data$Zooplankton, SizeClass=="Macro" & CPUE==0 & !is.na(Volume) & Source=="EMP" & Taxlifestage%in%c("Hyperacanthomysis longirostris Adult", "Neomysis mercedis Adult"))$BPUE), 0) }) test_that("The only Macro masses are from EMP", { - expect_equal(unique(filter(Data$Zooplankton, SizeClass=="Macro" & !is.na(BPUE))$Source), "EMP") + expect_equal(unique(dplyr::filter(Data$Zooplankton, SizeClass=="Macro" & !is.na(BPUE))$Source), "EMP") }) test_that("The only Macro species with biomasses are those we expect", { - expect_setequal(unique(filter(Data$Zooplankton, SizeClass=="Macro" & !is.na(BPUE))$Taxlifestage), + expect_setequal(unique(dplyr::filter(Data$Zooplankton, SizeClass=="Macro" & !is.na(BPUE))$Taxlifestage), c("Hyperacanthomysis longirostris Adult", "Neomysis mercedis Adult")) })