Handle errors reading file (#152)

Author: chopkinsmade

src/hooks/presidio/path_filter.py

@@ -18,6 +18,7 @@ class PathScanStatus(Enum):
     EXCLUDED = 2
     PASSED = 3
     FAILED = 4
+    ERRORED = 5
 
 
 class PathFilter:

src/hooks/presidio/scanner.py

@@ -51,6 +51,7 @@ def __init__(self, results: List[PathScanResult] = []) -> None:
         self.paths_containing_personal_data: List[PathScanResult] = []
         self.paths_skipped: List[PathScanResult] = []
         self.paths_excluded: List[PathScanResult] = []
+        self.paths_errored: List[PathScanResult] = []
         self.add_path_scan_results(results)
 
     def add_path_scan_results(self, scan_results: List[PathScanResult]):
@@ -70,6 +71,9 @@ def add_path_scan_result(self, scan_result: PathScanResult):
         if scan_result.status == PathScanStatus.SKIPPED:
             self.paths_skipped.append(scan_result)
 
+        if scan_result.status == PathScanStatus.ERRORED:
+            self.paths_errored.append(scan_result)
+
     def __str__(self) -> str:
         with StringIO() as output_buffer:
             output_buffer.write("--------PERSONAL DATA SCAN SUMMARY--------")
@@ -94,6 +98,13 @@ def __str__(self) -> str:
                     paths_without_issues_table.add_row([valid_path.path])
                 output_buffer.write(str(paths_without_issues_table))
 
+            if self.paths_errored:
+                output_buffer.write("\n\nFILES ERRORED\n")
+                errored_paths_table = PrettyTable(["Path"])
+                for errored_path in self.paths_errored:
+                    errored_paths_table.add_row([errored_path.path])
+                output_buffer.write(str(errored_paths_table))
+
             if self.paths_containing_personal_data:
                 output_buffer.write("\n\nFILES CONTAINING PERSONAL DATA\n")
 
@@ -155,29 +166,33 @@ def _scan_content(self, analyzer: AnalyzerEngine, entities: List[str], content:
     async def _scan_path(
         self, analyzer: AnalyzerEngine, entities: List[str], file_path: str, exclusions: List[re.Pattern[str]]
     ) -> PathScanResult:
-        sources = PathFilter()
-
-        invalid_check_result = await sources._check_is_path_invalid(file_path, exclusions)
-        if invalid_check_result is not None:
-            return PathScanResult(file_path, invalid_check_result)
-
-        file_extension = Path(file_path).suffix.lower()
-        async with await open_file(file_path, "r", encoding="utf-8") as fs:
-            results: List[PersonalDataDetection] = []
-            if file_extension in self.LINE_BY_LINE_FILE_EXTENSIONS:
-                logger.debug("Scanning file %s line by line", file_path)
-                async for line in fs:
-                    results.extend(self._scan_content(analyzer, entities, line.rstrip()))
-            else:
-                contents = await fs.read()
-                logger.debug("Scanning file %s by reading all contents", file_path)
-                results.extend(self._scan_content(analyzer, entities, contents))
-
-            return PathScanResult(
-                file_path,
-                status=PathScanStatus.PASSED if len(results) == 0 else PathScanStatus.FAILED,
-                results=results,
-            )
+        try:
+            sources = PathFilter()
+
+            invalid_check_result = await sources._check_is_path_invalid(file_path, exclusions)
+            if invalid_check_result is not None:
+                return PathScanResult(file_path, invalid_check_result)
+
+            file_extension = Path(file_path).suffix.lower()
+            async with await open_file(file_path, "r", encoding="utf-8") as fs:
+                results: List[PersonalDataDetection] = []
+                if file_extension in self.LINE_BY_LINE_FILE_EXTENSIONS:
+                    logger.debug("Scanning file %s line by line", file_path)
+                    async for line in fs:
+                        results.extend(self._scan_content(analyzer, entities, line.rstrip()))
+                else:
+                    contents = await fs.read()
+                    logger.debug("Scanning file %s by reading all contents", file_path)
+                    results.extend(self._scan_content(analyzer, entities, contents))
+
+                return PathScanResult(
+                    file_path,
+                    status=PathScanStatus.PASSED if len(results) == 0 else PathScanStatus.FAILED,
+                    results=results,
+                )
+        except Exception:
+            logger.exception("The file scanner failed to read file %s", file_path, stack_info=True)
+            return PathScanResult(file_path, status=PathScanStatus.ERRORED)
 
     async def scan(
         self,

tests/unit/hooks/presidio/test_scanner.py

@@ -65,6 +65,17 @@ async def test_scan_path_scans_file_contents_for_file_extensions_with_expected_r
                 mock_scan_content.assert_called_once_with(ANY, ANY, contents)
                 assert pickle.dumps(result) == pickle.dumps(expected_scan_result)
 
+    async def test_scan_path_handles_exception(self):
+        async with NamedTemporaryFile(suffix="file1.csv", mode="w+t") as tf:
+            with patch.object(PresidioScanner, "_scan_content") as mock_scan_content:
+                mock_scan_content.side_effect = Exception()
+                contents = "Error reading this file"
+                await tf.write(contents)
+                await tf.seek(0)
+
+                result = await PresidioScanner()._scan_path(MagicMock(), [], tf.name, [])
+                assert result.status == PathScanStatus.ERRORED
+
     def test_scan_content_returns_detections_list_when_path_has_personal_data(self):
         contents = "I have personal data"