ENH: Dream3dImportParameter supports additional import options. (#1230)

* Add Include/Exclude import policies to Dream3dImportParameter.
* Add 'All' enumeration option, and switch default to 'IncludeList'.
* Switch default to 'All' and require use of ImportData constructor
* Modify filter to allow singular data paths without including parent paths.

This change allows the filter to handle "Geometry/CellData/Fit" by itself
instead of having to also input "Geometry" and "Geometry/CellData".

This change will mainly benefit anyone running this filter without the GUI.

---------

Signed-off-by: Joey Kleingers <joey.kleingers@bluequartz.net>
Co-authored-by: Michael Jackson <mike.jackson@bluequartz.net>
Co-authored-by: Jared Duffey <jared.duffey@bluequartz.net>

Author: 3 people

src/Plugins/OrientationAnalysis/test/MergeTwinsTest.cpp

@@ -67,8 +67,8 @@ TEST_CASE("Reconstruction::MergeTwinsFilter: Valid Execution", "[Reconstruction]
     auto filter = filterList->createFilter(k_ReadDREAM3DFilterHandle);
     REQUIRE(nullptr != filter);
 
-    Dream3dImportParameter::ImportData parameter;
-    parameter.FilePath = fs::path(fmt::format("{}/neighbor_orientation_correlation.dream3d", unit_test::k_TestFilesDir));
+    auto filePath = fs::path(fmt::format("{}/neighbor_orientation_correlation.dream3d", unit_test::k_TestFilesDir));
+    Dream3dImportParameter::ImportData parameter(filePath);
 
     Arguments args;
     args.insertOrAssign(k_ImportFileData, std::make_any<Dream3dImportParameter::ImportData>(parameter));

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadDREAM3DFilter.cpp

@@ -4,6 +4,7 @@
 #include "simplnx/Filter/Actions/ImportH5ObjectPathsAction.hpp"
 #include "simplnx/Parameters/Dream3dImportParameter.hpp"
 #include "simplnx/Parameters/StringParameter.hpp"
+#include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp"
 #include "simplnx/Utilities/Parsing/HDF5/Readers/FileReader.hpp"
 
 #include "simplnx/Utilities/SIMPLConversion.hpp"
@@ -14,6 +15,56 @@ namespace
 {
 constexpr nx::core::int32 k_NoImportPathError = -1;
 constexpr nx::core::int32 k_FailedOpenFileReaderError = -25;
+constexpr nx::core::int32 k_UnsupportedPathImportPolicyError = -51;
+
+std::vector<nx::core::DataPath> ExpandSelectedPathsToAncestors(const std::vector<nx::core::DataPath>& selectedPaths)
+{
+  std::vector<nx::core::DataPath> finalDataPaths;
+  for(const auto& dataPath : selectedPaths)
+  {
+    auto pathVector = dataPath.getPathVector();
+    for(size_t i = 1; i <= dataPath.getLength(); ++i)
+    {
+      auto dataPathPart = nx::core::DataPath(std::vector<std::string>(pathVector.begin(), pathVector.begin() + i));
+      if(std::find(finalDataPaths.begin(), finalDataPaths.end(), dataPathPart) == finalDataPaths.end())
+      {
+        finalDataPaths.push_back(dataPathPart);
+      }
+    }
+  }
+
+  return finalDataPaths;
+}
+
+std::vector<nx::core::DataPath> ExpandSelectedPathsToDescendants(const std::vector<nx::core::DataPath>& selectedPaths, const std::vector<nx::core::DataPath>& allPaths)
+{
+  std::vector<nx::core::DataPath> expandedDataPaths = selectedPaths;
+  for(const auto& dataPath : selectedPaths)
+  {
+    for(const auto& candidateDataPath : allPaths)
+    {
+      if(candidateDataPath.getLength() <= dataPath.getLength())
+      {
+        continue;
+      }
+
+      bool isEqual = true;
+      for(size_t i = 0; i < dataPath.getPathVector().size(); ++i)
+      {
+        if(dataPath.getPathVector()[i] != candidateDataPath.getPathVector()[i])
+        {
+          isEqual = false;
+        }
+      }
+      if(isEqual)
+      {
+        expandedDataPaths.push_back(candidateDataPath);
+      }
+    }
+  }
+
+  return expandedDataPaths;
+}
 } // namespace
 
 namespace nx::core
@@ -53,7 +104,8 @@ Parameters ReadDREAM3DFilter::parameters() const
 {
   Parameters params;
   params.insertSeparator(Parameters::Separator{"Input Parameter(s)"});
-  params.insert(std::make_unique<Dream3dImportParameter>(k_ImportFileData, "Import File Path", "The HDF5 file path the DataStructure should be imported from.", Dream3dImportParameter::ImportData()));
+  params.insert(
+      std::make_unique<Dream3dImportParameter>(k_ImportFileData, "Import File Path", "The HDF5 file path the DataStructure should be imported from.", Dream3dImportParameter::ImportData({})));
   return params;
 }
 
@@ -75,18 +127,79 @@ IFilter::PreflightResult ReadDREAM3DFilter::preflightImpl(const DataStructure& d
   auto importData = args.value<Dream3dImportParameter::ImportData>(k_ImportFileData);
   if(importData.FilePath.empty())
   {
-    return {nonstd::make_unexpected(std::vector<Error>{Error{k_NoImportPathError, "Import file path not provided."}})};
+    return {MakeErrorResult<OutputActions>(k_NoImportPathError, "Import file path not provided.")};
   }
   nx::core::HDF5::FileReader fileReader(importData.FilePath);
   if(!fileReader.isValid())
   {
-    return {nonstd::make_unexpected(std::vector<Error>{Error{k_FailedOpenFileReaderError, "Failed to open the HDF5 file at the specified path."}})};
+    return {MakeErrorResult<OutputActions>(k_FailedOpenFileReaderError, "Failed to open the HDF5 file at the specified path.")};
+  }
+
+  Result<OutputActions> result;
+  OutputActions& actions = result.value();
+
+  Result<DataStructure> dataStructureResult = DREAM3D::ImportDataStructureFromFile(fileReader, true);
+  if(dataStructureResult.invalid())
+  {
+    return {ConvertResultTo<OutputActions>(ConvertResult(std::move(dataStructureResult)), {})};
+  }
+  auto importedDataStructure = dataStructureResult.value();
+
+  if(importData.ImportPolicy == Dream3dImportParameter::PathImportPolicy::IncludeList)
+  {
+    auto allDataPaths = importedDataStructure.getAllDataPaths();
+    std::vector<DataPath> selectedDataPaths = ExpandSelectedPathsToDescendants(importData.DataPaths, allDataPaths);
+    selectedDataPaths = ExpandSelectedPathsToAncestors(selectedDataPaths);
+
+    if(selectedDataPaths.empty())
+    {
+      result.warnings().push_back(
+          Warning{-10, "The import policy is set to 'Include List' and the file paths list is empty.  This will result in no data being imported.  Is this what you meant to do?"});
+    }
+
+    actions.appendAction(std::make_unique<ImportH5ObjectPathsAction>(importData.FilePath, selectedDataPaths));
+  }
+  else if(importData.ImportPolicy == Dream3dImportParameter::PathImportPolicy::ExcludeList || importData.ImportPolicy == Dream3dImportParameter::PathImportPolicy::All)
+  {
+    auto allDataPaths = importedDataStructure.getAllDataPaths();
+    if(importData.ImportPolicy == Dream3dImportParameter::PathImportPolicy::ExcludeList)
+    {
+      if(importData.DataPaths.empty())
+      {
+        result.warnings().push_back(Warning{-11,
+                                            "The import policy is set to 'Exclude List' and the file paths list is empty.  This will result in all data being imported.  You can accomplish the same "
+                                            "result by setting the import policy to 'All'."});
+        actions.appendAction(std::make_unique<ImportH5ObjectPathsAction>(importData.FilePath, allDataPaths));
+      }
+      else
+      {
+        auto expandedDataPaths = ExpandSelectedPathsToDescendants(importData.DataPaths, allDataPaths);
+
+        // Erase the expandedDataPaths from allPaths to create finalDataPaths
+        std::vector<nx::core::DataPath> finalDataPaths = allDataPaths;
+        for(const auto& dataPath : expandedDataPaths)
+        {
+          auto iter = std::find(finalDataPaths.begin(), finalDataPaths.end(), dataPath);
+          if(iter != finalDataPaths.end())
+          {
+            finalDataPaths.erase(iter);
+          }
+        }
+
+        actions.appendAction(std::make_unique<ImportH5ObjectPathsAction>(importData.FilePath, finalDataPaths));
+      }
+    }
+    else
+    {
+      actions.appendAction(std::make_unique<ImportH5ObjectPathsAction>(importData.FilePath, allDataPaths));
+    }
+  }
+  else
+  {
+    return {MakeErrorResult<OutputActions>(k_UnsupportedPathImportPolicyError, "The chosen PathImportPolicy is not supported by this filter.  Please contact the developers.")};
   }
 
-  OutputActions actions;
-  auto action = std::make_unique<ImportH5ObjectPathsAction>(importData.FilePath, importData.DataPaths);
-  actions.appendAction(std::move(action));
-  return {std::move(actions)};
+  return {result};
 }
 
 Result<> ReadDREAM3DFilter::executeImpl(DataStructure& dataStructure, const Arguments& args, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler,

src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp

@@ -185,9 +185,9 @@ Pipeline CreateImportPipeline()
   Pipeline pipeline("Import DREAM3D Pipeline");
   {
     Arguments args;
-    Dream3dImportParameter::ImportData importData;
-    importData.FilePath = GetExportDataPath();
-    importData.DataPaths = std::vector<DataPath>{DataPath({DataNames::k_Group1Name}), DataPath({DataNames::k_ArrayName})};
+    auto filePath = GetExportDataPath();
+    std::vector<DataPath> dataPaths = std::vector<DataPath>{DataPath({DataNames::k_Group1Name}), DataPath({DataNames::k_ArrayName})};
+    Dream3dImportParameter::ImportData importData(filePath, Dream3dImportParameter::PathImportPolicy::IncludeList, dataPaths);
     args.insert("import_data_object", importData);
     pipeline.push_back(k_ImportD3DHandle, args);
   }
@@ -241,17 +241,13 @@ Pipeline CreateMultiImportPipeline()
   Pipeline pipeline("Import DREAM3D Pipeline");
   {
     Arguments args;
-    Dream3dImportParameter::ImportData importData;
-    importData.FilePath = GetMultiExportDataPath1();
-    importData.DataPaths = std::vector<DataPath>{DataPath({DataNames::k_Group1Name})};
+    Dream3dImportParameter::ImportData importData(GetMultiExportDataPath1(), Dream3dImportParameter::PathImportPolicy::IncludeList, std::vector<DataPath>{DataPath({DataNames::k_Group1Name})});
     args.insert("import_data_object", importData);
     pipeline.push_back(k_ImportD3DHandle, args);
   }
   {
     Arguments args;
-    Dream3dImportParameter::ImportData importData;
-    importData.FilePath = GetMultiExportDataPath2();
-    importData.DataPaths = std::vector<DataPath>{DataPath({DataNames::k_Group2Name})};
+    Dream3dImportParameter::ImportData importData(GetMultiExportDataPath2(), Dream3dImportParameter::PathImportPolicy::IncludeList, std::vector<DataPath>{DataPath({DataNames::k_Group2Name})});
     args.insert("import_data_object", importData);
     pipeline.push_back(k_ImportD3DHandle, args);
   }
@@ -282,7 +278,7 @@ TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test")
   {
     auto fileData = CreateFileData();
     Result<HDF5::FileWriter> result = HDF5::FileWriter::CreateFile(GetIODataPath());
-    REQUIRE(result.valid());
+    SIMPLNX_RESULT_REQUIRE_VALID(result);
 
     auto writeResult = DREAM3D::WriteFile(result.value(), fileData);
     SIMPLNX_RESULT_REQUIRE_VALID(writeResult);
@@ -408,10 +404,107 @@ TEST_CASE("DREAM3DFileTest: Existing Data Objects Test")
 
     ReadDREAM3DFilter filter;
     Arguments args;
-    Dream3dImportParameter::ImportData importData;
-    importData.FilePath = fs::path(fmt::format("{}/Small_IN100.dream3d", unit_test::k_TestFilesDir));
+    Dream3dImportParameter::ImportData importData(fs::path(fmt::format("{}/Small_IN100.dream3d", unit_test::k_TestFilesDir)));
     args.insert(ReadDREAM3DFilter::k_ImportFileData, importData);
     auto executeResult = filter.execute(ds, args);
     SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result);
   }
 }
+
+TEST_CASE("DREAM3DFileTest: Path Import Policy Tests")
+{
+  const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_CMakeExecutable, nx::core::unit_test::k_TestFilesDir, "Small_IN100_dream3d_v2.tar.gz", "Small_IN100.dream3d");
+  auto filePath = fs::path(fmt::format("{}/Small_IN100.dream3d", unit_test::k_TestFilesDir));
+  DataStructure dataStructure;
+  ReadDREAM3DFilter filter;
+  Arguments args;
+
+  SECTION("All")
+  {
+    Dream3dImportParameter::ImportData importData(filePath);
+    args.insert(ReadDREAM3DFilter::k_ImportFileData, importData);
+    auto executeResult = filter.execute(dataStructure, args);
+    SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result);
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData", "Confidence Index"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData", "EulerAngles"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData", "Fit"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData", "Image Quality"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData", "Phases"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData", "SEM Signal"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData", "CrystalStructures"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData", "LatticeConstants"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData", "MaterialName"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredXDataContainer"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredXDataContainer", "Cell Data"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredXDataContainer", "Cell Data", "Confidence Index"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredXDataContainer", "Cell Data", "EulerAngles"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredXDataContainer", "Cell Data", "Fit"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredXDataContainer", "Cell Data", "Image Quality"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredXDataContainer", "Cell Data", "Phases"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredXDataContainer", "Cell Data", "SEM Signal"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredYDataContainer"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredYDataContainer", "Cell Data"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredYDataContainer", "Cell Data", "Confidence Index"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredYDataContainer", "Cell Data", "EulerAngles"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredYDataContainer", "Cell Data", "Fit"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredYDataContainer", "Cell Data", "Image Quality"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredYDataContainer", "Cell Data", "Phases"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredYDataContainer", "Cell Data", "SEM Signal"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredZDataContainer"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredZDataContainer", "Cell Data"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredZDataContainer", "Cell Data", "Confidence Index"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredZDataContainer", "Cell Data", "EulerAngles"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredZDataContainer", "Cell Data", "Fit"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredZDataContainer", "Cell Data", "Image Quality"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredZDataContainer", "Cell Data", "Phases"})));
+    REQUIRE(dataStructure.containsData(DataPath({"MirroredZDataContainer", "Cell Data", "SEM Signal"})));
+  }
+  SECTION("Include List - Leaf Node")
+  {
+    Dream3dImportParameter::ImportData importData(filePath, Dream3dImportParameter::PathImportPolicy::IncludeList, {DataPath({"DataContainer", "CellData", "Confidence Index"})});
+    args.insert(ReadDREAM3DFilter::k_ImportFileData, importData);
+    auto executeResult = filter.execute(dataStructure, args);
+    SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result);
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData", "Confidence Index"})));
+    REQUIRE(!dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData"})));
+  }
+  SECTION("Include List - Parent Node")
+  {
+    Dream3dImportParameter::ImportData importData(filePath, Dream3dImportParameter::PathImportPolicy::IncludeList, {DataPath({"DataContainer", "CellData"})});
+    args.insert(ReadDREAM3DFilter::k_ImportFileData, importData);
+    auto executeResult = filter.execute(dataStructure, args);
+    SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result);
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData", "Confidence Index"})));
+    REQUIRE(!dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData"})));
+  }
+  SECTION("Exclude List - Parent Node")
+  {
+    Dream3dImportParameter::ImportData importData(filePath, Dream3dImportParameter::PathImportPolicy::ExcludeList, {DataPath({"DataContainer", "CellData"})});
+    args.insert(ReadDREAM3DFilter::k_ImportFileData, importData);
+    auto executeResult = filter.execute(dataStructure, args);
+    SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result);
+    REQUIRE(!dataStructure.containsData(DataPath({"DataContainer", "CellData"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData", "CrystalStructures"})));
+  }
+
+  SECTION("Exclude List - Leaf Node")
+  {
+    Dream3dImportParameter::ImportData importData(filePath, Dream3dImportParameter::PathImportPolicy::ExcludeList, {DataPath({"DataContainer", "CellData", "Confidence Index"})});
+    args.insert(ReadDREAM3DFilter::k_ImportFileData, importData);
+    auto executeResult = filter.execute(dataStructure, args);
+    SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result);
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData"})));
+    REQUIRE(!dataStructure.containsData(DataPath({"DataContainer", "CellData", "Confidence Index"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellData", "Fit"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData"})));
+    REQUIRE(dataStructure.containsData(DataPath({"DataContainer", "CellEnsembleData", "CrystalStructures"})));
+  }
+}