Merge pull request #747 from ldecicco-USGS/develop

Develop

Author: ldecicco-USGS

.github/workflows/R-CMD-check.yaml

@@ -21,10 +21,6 @@ jobs:
       matrix:
         config:
           - {os: macos-latest,   r: 'release'}
-          - {os: windows-latest, r: 'release'}
-          - {os: ubuntu-latest,   r: 'devel', http-user-agent: 'release'}
-          - {os: ubuntu-latest,   r: 'release'}
-          - {os: ubuntu-latest,   r: 'oldrel-1'}
 
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pkgdown.yaml

@@ -30,15 +30,15 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
       CUSTOM_DR_UA: 'GitHub_CI'
     steps:
-      - uses: actions/checkout@c0a81a463886bb75afe234e07a9fd5bb79219196
+      - uses: actions/checkout@581d62f320f2a4043a1ea6ac77290c60d27485cc
 
-      - uses: r-lib/actions/setup-pandoc@52330cc136b963487918a8867f948ddf954e9e63
+      - uses: r-lib/actions/setup-pandoc@473c68190595b311a74f208fba61a8d8c0d4c247
 
-      - uses: r-lib/actions/setup-r@52330cc136b963487918a8867f948ddf954e9e63
+      - uses: r-lib/actions/setup-r@473c68190595b311a74f208fba61a8d8c0d4c247
         with:
           use-public-rspm: true
 
-      - uses: r-lib/actions/setup-r-dependencies@52330cc136b963487918a8867f948ddf954e9e63
+      - uses: r-lib/actions/setup-r-dependencies@473c68190595b311a74f208fba61a8d8c0d4c247
         with:
           extra-packages: |
             any::pkgdown
@@ -58,7 +58,7 @@ jobs:
           file.copy(from = "./public/articles/logo.png",to = "./public/reference/logo.png")
         shell: Rscript {0}
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@64bcae551a7b18bcb9a09042ddf1960979799187
+        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa
         with:
           path: ./public
   # Deployment job
@@ -71,5 +71,5 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@73e62e651178eeba977de2dc9f4c7645b3d01015
+        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e
 

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: dataRetrieval
 Type: Package
 Title: Retrieval Functions for USGS and EPA Hydrology and Water Quality Data
-Version: 2.7.17.9001
+Version: 2.7.18
 Authors@R: c(
     person("Laura", "DeCicco", role = c("aut","cre"),
             email = "[email protected]",

R/getWebServiceData.R

@@ -52,29 +52,6 @@ getWebServiceData <- function(obs_url, ...) {
     content <- gsub(" ", "", tolower(headerInfo$`content-type`))
     if (content %in% return_content) {
       returnedDoc <- httr2::resp_body_string(returnedList)
-      trys <- 1
-      if (all(grepl("ERROR: INCOMPLETE DATA", returnedDoc))) {
-        
-        while(trys <= 3){
-          message("Trying again!")
-          obs_url <- httr2::req_url_query(obs_url, 
-                                          try = trys)
-          returnedList <- httr2::req_perform(obs_url)
-          
-          good <- check_non_200s(returnedList)
-          if(good){
-            returnedDoc <- httr2::resp_body_string(returnedList)
-          }
-          if (all(grepl("ERROR: INCOMPLETE DATA", returnedDoc))) {
-            trys <- trys + 1
-          } else {
-            trys <- 100
-          }
-        } 
-      }
-
-    # } else if (headerInfo$`content-type` %in% return_raw) {
-    #   returnedDoc <- httr2::resp_body_raw(returnedList)
     } else if (content %in% return_readLines) {
       returnedList <- httr2::resp_body_string(returnedList)
       txt <- readLines(returnedList$content)

R/importWQP.R

@@ -67,6 +67,10 @@ importWQP <- function(obs_url, tz = "UTC",
 
   attr(retval, 'spec') <- NULL
 
+  if(any(grepl("ERROR: INCOMPLETE DATA", retval[1,]))){
+    stop(retval[[1]])
+  }
+  
   # this is only needed for legacy
   names(retval)[grep("/", names(retval))] <- gsub("/", ".", names(retval)[grep("/", names(retval))])
 

R/readWQPdata.R

@@ -43,8 +43,7 @@
 #'
 #' @param \dots see \url{https://www.waterqualitydata.us/webservices_documentation} for a complete list of options.
 #' A list of arguments can also be supplied. For more information see the above 
-#' description for this help file. If no "service" argument is supplied, it
-#' will default to "ResultWQX3". One way to figure out how to construct a WQP query is to go to the "Advanced" 
+#' description for this help file. One way to figure out how to construct a WQP query is to go to the "Advanced" 
 #' form in the Water Quality Portal. Use the form to discover what parameters are available. Once the query is 
 #' set in the form, scroll down to the "Query URL". You will see the parameters
 #' after "https://www.waterqualitydata.us/#". For example, if you chose "Nutrient"
@@ -86,12 +85,21 @@
 #' @examplesIf is_dataRetrieval_user()
 #' \donttest{
 #' 
+#' # Legacy:
 #' nameToUse <- "pH"
 #' pHData <- readWQPdata(siteid = "USGS-04024315", 
 #'                       characteristicName = nameToUse)
 #' ncol(pHData)
 #' attr(pHData, "siteInfo")
 #' attr(pHData, "queryTime")
+#' attr(pHData, "url")
+#' 
+#' # WQX3:
+#' pHData_wqx3 <- readWQPdata(siteid = "USGS-04024315", 
+#'                       characteristicName = nameToUse,
+#'                       service = "ResultWQX3",
+#'                       dataProfile = "basicPhysChem")
+#' attr(pHData_wqx3, "url")
 #'
 #' # More examples:
 #' # querying by county
@@ -101,11 +109,30 @@
 #'   characteristicName = "Nitrogen"
 #' )
 #' 
+#' attr(DeWitt, "url")
+#' 
+#' DeWitt_wqx3 <- readWQPdata(
+#'   statecode = "Illinois",
+#'   countycode = "DeWitt",
+#'   characteristicName = "Nitrogen",
+#'   service = "ResultWQX3",
+#'   dataProfile = "basicPhysChem",  
+#'   ignore_attributes = TRUE)
+#' 
+#' attr(DeWitt_wqx3, "url")
+#' 
 #' # Data profile: "Sampling Activity"
 #' activity <- readWQPdata(
 #'   siteid = "USGS-04024315",
 #'   service = "Activity"
 #' )
+#' attr(activity, "url")
+#' 
+#' activity_wqx3 <- readWQPdata(
+#'   siteid = "USGS-04024315",
+#'   service = "ActivityWQX3"
+#' )
+#' attr(activity_wqx3, "url")
 #' 
 #' Dane_activity <- readWQPdata(
 #'   statecode = "Wisconsin",
@@ -114,14 +141,20 @@
 #'   startDateHi = "2023-12-31",
 #'   service = "Activity"
 #' )
+#' attr(Dane_activity, "url")
+#' 
+#' Dane_activity_wqx3 <- readWQPdata(
+#'   statecode = "Wisconsin",
+#'   countycode = "Dane",
+#'   startDateLo = "2023-01-01",
+#'   startDateHi = "2023-12-31",
+#'   service = "ActivityWQX3"
+#' )
+#' attr(Dane_activity_wqx3, "url")
 #' 
 #' ########################################################
 #' # Additional examples:
 #'
-#' pHData_legacy <- readWQPdata(siteid = "USGS-04024315", 
-#'                       characteristicName = nameToUse,
-#'                       service = "Result",
-#'                       dataProfile = "narrowResult")
 #'  
 #' # Data profiles: "Organization Data" 
 #' org_data <- readWQPdata(
@@ -157,6 +190,7 @@
 #'   dataProfile = "biological",
 #'   service = "Result"
 #' )
+#' 
 #'
 #' # Data profiles: "Sample Results (narrow)" 
 #' samp_narrow <- readWQPdata(
@@ -165,6 +199,13 @@
 #'   dataProfile = "narrowResult"
 #' )
 #'
+#' samp_narrow_wqx3 <- readWQPdata(
+#'   siteid = "USGS-04024315",
+#'   service = "ResultWQX3",
+#'   dataProfile = "narrow"
+#' )
+#'
+#'
 #' # Data profiles: "Sampling Activity"  
 #' samp_activity <- readWQPdata(
 #'   siteid = "USGS-04024315",
@@ -289,6 +330,29 @@ readWQPdata <- function(...,
 
 create_WQP_attributes <- function(retval, ...){
 
+  col_legacy <- c("CharacteristicName", #legacy
+                  "ResultMeasure.MeasureUnitCode", 
+                  "ResultSampleFractionText")
+  col_wqx3 <- c("Result_Characteristic", #wqx3
+                "Result_MeasureUnit", 
+                "Result_SampleFraction")
+  if (all(col_legacy %in% names(retval))) {
+    retvalVariableInfo <- retval[, col_legacy]
+    retvalVariableInfo <- unique(retvalVariableInfo)
+    names(retvalVariableInfo) <- c("characteristicName",
+                                   "param_units",
+                                   "valueType")
+
+    attr(retval, "variableInfo") <- retvalVariableInfo
+  } else if(all(col_wqx3 %in% names(retval))){
+    retvalVariableInfo <- retval[, col_wqx3]
+    retvalVariableInfo <- unique(retvalVariableInfo)
+    names(retvalVariableInfo) <- c("characteristicName",
+                                   "param_units",
+                                   "valueType")
+    
+    attr(retval, "variableInfo") <- retvalVariableInfo
+  }
 
   siteInfo <- suppressWarnings(whatWQPsites(..., legacy = attr(retval, "legacy")))
   attr(retval, "siteInfo") <- siteInfo
@@ -301,8 +365,6 @@ create_WQP_attributes <- function(retval, ...){
     attr(retval, "queryTime") <- Sys.time()
   }
 
-  
-  
   return(retval)
 }
 

R/readWQPdots.R

@@ -77,6 +77,8 @@ readWQPdots <- function(..., legacy = TRUE) {
     # in WQP, but if they also then request county codes,
     # it gets really confusing, and the WQP developers don't recommend.
     values["countycode"] <- countyCdLookup(stCd, values["countycode"], "fips")
+    # WQP doesn't need or want both state and county code:
+    values["statecode"] <- NULL
   }
 
   if(!"mimeType" %in% names(values)){

R/readWQPqw.R

@@ -55,13 +55,24 @@
 #' 
 #' attr(rawPcode, "siteInfo")
 #' attr(rawPcode, "queryTime")
-#' 
+#' attr(rawPcode, "url") 
 #' 
 #' rawCharacteristicName <- readWQPqw("WIDNR_WQX-10032762", "Specific conductance", "", "")
-#' rawPHsites <- readWQPqw(c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"), "pH", "", "")
+#' pHsites_legacy <- readWQPqw(c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"),
+#'                         "pH", "", "")
+#' ncol(pHsites_legacy)
+#' attr(pHsites_legacy, "url")
+#'  
+#' pHsites_modern <- readWQPqw(c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"),
+#'                         "pH", "", "", legacy = FALSE)
+#' ncol(pHsites_modern)
+#' attr(pHsites_modern, "url")
+#'           
 #' nwisEx <- readWQPqw("USGS-04024000", c("34247", "30234", "32104", "34220"), "", "2022-12-20")
 #'
-#' SC <- readWQPqw(siteNumbers = "USGS-05288705", parameterCd = "00300", convertType = FALSE)
+#' DO <- readWQPqw(siteNumbers = "USGS-05288705", 
+#'                 parameterCd = "00300",
+#'                 convertType = FALSE)
 #' }
 readWQPqw <- function(siteNumbers,
                       parameterCd,

README.Rmd

@@ -29,17 +29,9 @@ library(dataRetrieval)
 
 The `dataRetrieval` package was created to simplify the process of loading hydrologic data into the R environment. It is designed to retrieve the major data types of U.S. Geological Survey (USGS) hydrology data that are available on the Web, as well as data from the Water Quality Portal (WQP), which currently houses water quality data from the Environmental Protection Agency (EPA), U.S. Department of Agriculture (USDA), and USGS. Direct USGS data is obtained from a service called the National Water Information System (NWIS). 
 
-:warning: USGS discrete water-quality data availability and format are changing. Beginning March 11, 2024 USGS data obtained from legacy services will not include new USGS data or recent updates to existing data. To view the status of changes in data availability and code functionality, visit: <https://doi-usgs.github.io/dataRetrieval/articles/Status.html>
-
-:warning: All documentation on <https://doi-usgs.github.io/dataRetrieval> assume you have the development version of dataRetrieval installed. To do that:
+# Introduction
 
-```{r eval=FALSE}
-library(remotes)
-install_github("DOI-USGS/dataRetrieval",
-               build_vignettes = TRUE, 
-               build_opts = c("--no-resave-data",
-                              "--no-manual"))
-```
+:warning: USGS discrete water-quality data availability and format are changing. Beginning March 11, 2024 USGS data obtained from legacy services will not include new USGS data or recent updates to existing data. To view the status of changes in data availability and code functionality, visit: <https://doi-usgs.github.io/dataRetrieval/articles/Status.html>
 
 If you have additional questions about these changes, email [email protected].