TADA_DefineCriteriaMethodology bug

**Is your feature request related to a problem? Please describe:**

```r
> Criteria <- TADA_DefineCriteriaMethodology(
+   AUMLRef_df_with_type,
+   AUMLRef = df_AUMLRef_final_with_water,
+   AU_UsesRef = UseAURef,
+   org_id = "MEDEP",
+   auto_assign = TRUE
+ )
TADA_DefineCriteriaMethodology: auto_assign = TRUE was selected but no MLSummaryRef. Generating TADA_MLSummary with default assignment.
TADA_DefineCriteriaMethodology: auto_assign = TRUE was selected. Running TADA_ParametersForAnalysis with default assignment.
TADA_DefineCriteriaMethodology: auto_assign = TRUE was selected. Running TADA_UsesForAnalysis with default assignment.
TADA_UsesForAnalysis: auto_assign == TRUE was selected, assigning all unique ATTAINS.UseName, by ATTAINS.OrganizationIdentifier, to any ATTAINS.ParameterName that an organization have not done assessments for in prior ATTAINS cycle. Please review carefully and Exclude rows as needed.
Error in `dplyr::select()` at EPATADA/R/ATTAINSCrosswalks.R:3731:7:
! Can't select columns that don't exist.
✖ Column `ATTAINS.OrganizationIdentifier` doesn't exist.
Run `rlang::last_trace()` to see where the error occurred.
Called from: signal_abort(cnd)
```

Reproducible code:

``` r
# # Query by tribal land area
# tribe <- TADA_TribalOptions(tribal_area_type = "Off-reservation Trust Lands")
# 
# TADA.penobscot <- TADA_DataRetrieval(
#   tribal_area_type = "Off-reservation Trust Lands",
#   tribe_name_parcel = "Penobscot Nation"
# )

# df <- TADA_DataRetrieval(
#   characteristicName = "Escherichia coli",
#   organization = "PENOBSCOTINDIANNATIONDNR",
#   maxrecs = 350000,
#   ask = TRUE,
#   applyautoclean = TRUE
# )

df_raw <- TADA_DataRetrieval(
  siteid = c(
    "PENOBSCOTINDIANNATIONDNR-130-BM1",
    "PENOBSCOTINDIANNATIONDNR-4-CH1",
    "PENOBSCOTINDIANNATIONDNR-5-DC1",
    "PENOBSCOTINDIANNATIONDNR-6-DP1",
    "PENOBSCOTINDIANNATIONDNR-76-DP3",
    "PENOBSCOTINDIANNATIONDNR-13-EM1",
    "PENOBSCOTINDIANNATIONDNR-14-EM2",
    "PENOBSCOTINDIANNATIONDNR-15-GB1",
    "PENOBSCOTINDIANNATIONDNR-16-GB2",
    "PENOBSCOTINDIANNATIONDNR-17-GB3",
    "PENOBSCOTINDIANNATIONDNR-18-GF1",
    "PENOBSCOTINDIANNATIONDNR-19-GF2",
    "PENOBSCOTINDIANNATIONDNR-113-GW1",
    "PENOBSCOTINDIANNATIONDNR-114-GW2",
    "PENOBSCOTINDIANNATIONDNR-115-GW3",
    "PENOBSCOTINDIANNATIONDNR-129-GWTR1",
    "PENOBSCOTINDIANNATIONDNR-20-LE1",
    "PENOBSCOTINDIANNATIONDNR-21-LE2",
    "PENOBSCOTINDIANNATIONDNR-22-LE3",
    "PENOBSCOTINDIANNATIONDNR-23-LI1",
    "PENOBSCOTINDIANNATIONDNR-24-LT1",
    "PENOBSCOTINDIANNATIONDNR-25-MD1",
    "PENOBSCOTINDIANNATIONDNR-26-MD2",
    "PENOBSCOTINDIANNATIONDNR-27-MI1",
    "PENOBSCOTINDIANNATIONDNR-28-MI2",
    "PENOBSCOTINDIANNATIONDNR-29-MI3",
    "PENOBSCOTINDIANNATIONDNR-30-MS1",
    "PENOBSCOTINDIANNATIONDNR-31-MW1",
    "PENOBSCOTINDIANNATIONDNR-32-MW2",
    "PENOBSCOTINDIANNATIONDNR-33-NL1",
    "PENOBSCOTINDIANNATIONDNR-117-OR1",
    "PENOBSCOTINDIANNATIONDNR-34-OT1",
    "PENOBSCOTINDIANNATIONDNR-35-OT2",
    "PENOBSCOTINDIANNATIONDNR-36-OT3",
    "PENOBSCOTINDIANNATIONDNR-37-PA1",
    "PENOBSCOTINDIANNATIONDNR-38-PA2",
    "PENOBSCOTINDIANNATIONDNR-39-PA3",
    "PENOBSCOTINDIANNATIONDNR-40-PA4",
    "PENOBSCOTINDIANNATIONDNR-41-PI1",
    "PENOBSCOTINDIANNATIONDNR-42-PI2",
    "PENOBSCOTINDIANNATIONDNR-43-PI3",
    "PENOBSCOTINDIANNATIONDNR-44-PS1",
    "PENOBSCOTINDIANNATIONDNR-46-SH2",
    "PENOBSCOTINDIANNATIONDNR-47-SL1",
    "PENOBSCOTINDIANNATIONDNR-81-TCOS1",
    "PENOBSCOTINDIANNATIONDNR-87-THOB",
    "PENOBSCOTINDIANNATIONDNR-135-TPIR14",
    "PENOBSCOTINDIANNATIONDNR-105-TPIR15",
    "PENOBSCOTINDIANNATIONDNR-123-TPIR16",
    "PENOBSCOTINDIANNATIONDNR-100-TPIR3",
    "PENOBSCOTINDIANNATIONDNR-103-TPIR6",
    "PENOBSCOTINDIANNATIONDNR-104-TPIR9",
    "PENOBSCOTINDIANNATIONDNR-86-TPOB",
    "PENOBSCOTINDIANNATIONDNR-91-TPUS1",
    "PENOBSCOTINDIANNATIONDNR-92-TPUS2",
    "PENOBSCOTINDIANNATIONDNR-131-VZI1",
    "PENOBSCOTINDIANNATIONDNR-132-VZTR1",
    "PENOBSCOTINDIANNATIONDNR-48-WB1",
    "PENOBSCOTINDIANNATIONDNR-49-WBU1",
    "PENOBSCOTINDIANNATIONDNR-50-WD1",
    "PENOBSCOTINDIANNATIONDNR-51-WD2",
    "PENOBSCOTINDIANNATIONDNR-75-WD3",
    "PENOBSCOTINDIANNATIONDNR-52-WE1",
    "PENOBSCOTINDIANNATIONDNR-53-WE2",
    "PENOBSCOTINDIANNATIONDNR-54-WE3",
    "PENOBSCOTINDIANNATIONDNR-56-WL1"
  ),
  characteristicName = "Escherichia coli",
  maxrecs = 350000,
  ask = FALSE,
  applyautoclean = TRUE
)

# df_ML <- unique(df$MonitoringLocationIdentifier)
unique(df_raw$MonitoringLocationIdentifier)

# IDs present in df but missing from df2
# missing_from_df2 <- setdiff(df_ML, df2_ML)

# Quick summaries
# length(missing_from_df2)

########################################################################

# None available
# filter_df <- TADA_MediaFilter(df2, 
#                               clean = TRUE,
#                               surface_water = FALSE,
#                               ground_water = TRUE,
#                               sediment = TRUE,
#                               other = TRUE)

df_cleaned <- TADA_FindPotentialDuplicatesSingleOrg(df_raw)

# Select a small number of columns to review duplicates
TADA_TableExport(subset(
  df_cleaned,
  TADA.SingleOrgDupGroupID != "Not a duplicate",
  select = c(ResultIdentifier,
             ActivityStartDate,
             TADA.ActivityMediaName,
             TADA.MonitoringLocationIdentifier,
             TADA.CharacteristicName,
             TADA.ResultMeasureValue,
             TADA.ResultMeasure.MeasureUnitCode,
             ResultAnalyticalMethod.MethodName)
))

# Drop duplicates
df_cleaned <- dplyr::filter(df_cleaned, TADA.SingleOrgDup.Flag == "Unique")

# # Handle censored results (nondetects)
# # None available
# nondetects <- TADA_SimpleCensoredMethods(
#   filter_df,
#   nd_method = "multiplier",
#   nd_multiplier = 0.5,
#   od_method = "as-is",
#   od_multiplier = "null"
# )

# # Flag non-numeric result values. There are none
# df_filtered_NA <- TADA_ConvertSpecialChars(df_filtered, 
#                                         col = "TADA.ResultMeasureValue",
#                                         clean = FALSE, 
#                                         flaggedonly = TRUE)

# No issues found
df_cleaned <- TADA_RunKeyFlagFunctions(df_cleaned, clean = FALSE)

# Harmonize synonyms
df_cleaned <- TADA_HarmonizeSynonyms(df_cleaned)
unique(df_cleaned$TADA.ComparableDataIdentifier)

############################# mod 2
# review valid ATTAINS org IDs
ATTAINS_orgs <- rExpertQuery::EQ_DomainValues("org_id")

# Look, there are matches!
AUMLRef <- TADA_CreateATTAINSAUMLCrosswalk(  df_cleaned,
                                             org_id = "MEDEP", # optionally specify MEDEP
                                             return_nearest = TRUE,
                                             return_sf = TRUE)

# extract TADA_with_ATTAINS data frame from the list
AUMLRef_df <- AUMLRef$TADA_with_ATTAINS

library(sf)

df_AUMLRef_final <- unique(subset(
  sf::st_drop_geometry(AUMLRef_df),
  select = c(
    OrganizationIdentifier,
    ATTAINS.OrganizationIdentifier,
    TADA.MonitoringLocationIdentifier,
    ATTAINS.AssessmentUnitIdentifier
  )
))

TADA_TableExport(df_AUMLRef_final)

# Count unique
length(unique(df_raw$TADA.MonitoringLocationIdentifier))
length(unique(df_AUMLRef_final$TADA.MonitoringLocationIdentifier))

# Add water types
df_AUMLRef_final_withtypes = TADA_AssignUsesToWaterType(AUMLRef_df,
                                         org_id = "MEDEP",
                                         AUMLRef = df_AUMLRef_final)

# Join AUMLRef_df 
library(dplyr)

types_map <- df_AUMLRef_final_withtypes %>%
  select(`ATTAINS.OrganizationIdentifier`, `ATTAINS.WaterType`) %>%
  group_by(`ATTAINS.OrganizationIdentifier`) %>%
  summarise(`ATTAINS.WaterType` = paste(unique(na.omit(`ATTAINS.WaterType`)), collapse = "; "),
            .groups = "drop")

AUMLRef_df_with_type <- AUMLRef_df %>%
  left_join(types_map, by = "ATTAINS.OrganizationIdentifier")

#############
# Add water type from AUMLRef_df_with_type to AUMLRef_df
org_water_type <- AUMLRef_df_with_type %>%
  select(`ATTAINS.OrganizationIdentifier`, `ATTAINS.WaterType`) %>%
  group_by(`ATTAINS.OrganizationIdentifier`) %>%
  summarise(`ATTAINS.WaterType` = paste(unique(na.omit(`ATTAINS.WaterType`)), collapse = "; "),
            .groups = "drop")

df_AUMLRef_final_with_water <- df_AUMLRef_final %>%
  left_join(org_water_type, by = "ATTAINS.OrganizationIdentifier")

# Optional check
stopifnot(nrow(df_AUMLRef_final_with_water) == nrow(df_AUMLRef_final))
#############

# Assign Uses
UseAURef <- TADA_AssignUsesToAU(
  AUMLRef_df_with_type,
  AUMLRef = df_AUMLRef_final_with_water,
  org_id = "MEDEP"
)

TADA_TableExport(UseAURef)
# 
# # Can we create a function to add the missing Water Type info?
# colnames(df_AUMLRef_final_withtypes)
# colnames(df_AUMLRef_final)

Criteria <- TADA_DefineCriteriaMethodology(
  AUMLRef_df_with_type,
  AUMLRef = df_AUMLRef_final_with_water,
  AU_UsesRef = UseAURef,
  org_id = "MEDEP",
  auto_assign = TRUE
)

# TADA_spatial <- TADA_MakeSpatial(.data = AUMLRef_df, crs = 4326)
# 
# # Map, not sure why this is not working
# TADA_ViewATTAINS(TADA_spatial, ref_icons = TRUE)
# 
# TADA_ViewATTAINS(AUMLRef, ref_icons = TRUE)

# AUMLRef <- TADA_GetATTAINSAUMLCrosswalk(
#   org_id = "all",
#   batch_upload = TRUE,
#   api_key = NULL
# )
# 
# library(dplyr)
# 
# # Keep only needed columns (optionally ensure unique keys)
# lookup <- AUMLRef %>%
#   select(MS_LOCATION_ID, ASSESSMENT_UNIT_ID) %>%
#   distinct()
# 
# if (any(df_cleaned$TADA.MonitoringLocationIdentifier %in% lookup$MS_LOCATION_ID, na.rm = TRUE)) {
#   df_cleaned <- df_cleaned %>%
#     left_join(lookup, by = c("TADA.MonitoringLocationIdentifier" = "MS_LOCATION_ID"))
# } else {
#   message("No matches found; df_cleaned left unchanged.")
# }

# AUMLRef$MS_LOCATION_ID
# df_cleaned$TADA.MonitoringLocationIdentifier
# 
# AUMLRef <- TADA_CreateAUMLCrosswalk(
#   df_cleaned,
#   org_id = "all",
#   fill_ATTAINS_catch = TRUE,
#   return_nearest = TRUE,
#   batch_upload = FALSE,
# )
```

Navigation Menu

Uh oh!

TADA_DefineCriteriaMethodology bug #894

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Uh oh!

TADA_DefineCriteriaMethodology bug #894

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions