Add a TODOs sheet containing on REQUIRES_REVIEW resources in XLSX #1524

Signed-off-by: tdruez <[email protected]>

Author: tdruez

scanpipe/pipes/output.py

@@ -53,6 +53,7 @@
 from scancodeio import SCAN_NOTICE
 from scancodeio import __version__ as scancodeio_version
 from scanpipe.pipes import docker
+from scanpipe.pipes import flag
 from scanpipe.pipes import spdx
 
 scanpipe_app = apps.get_app_config("scanpipe")
@@ -96,6 +97,30 @@ def get_queryset(project, model_name):
     return querysets.get(model_name)
 
 
+TODO_FIELDS = [
+    "path",
+    "status",
+    "size",
+    "name",
+    "extension",
+    "programming_language",
+    "mime_type",
+    "tag",
+    "detected_license_expression",
+    "compliance_alert",
+    "project__name",
+]
+
+
+def get_todos_data(project):
+    """Return the list of Resources that requires review."""
+    return (
+        project.codebaseresources.files()
+        .filter(status=flag.REQUIRES_REVIEW)
+        .values(*TODO_FIELDS)
+    )
+
+
 def queryset_to_csv_file(queryset, fieldnames, output_file):
     """
     Output csv content generated from the provided `queryset` objects to the
@@ -346,8 +371,12 @@ def _add_xlsx_worksheet(workbook, worksheet_name, rows, fields):
 
     for row_index, record in enumerate(rows, start=1):
         row_errors = []
+        record_is_dict = isinstance(record, dict)
         for col_index, field in enumerate(fields):
-            value = getattr(record, field)
+            if record_is_dict:
+                value = record.get(field)
+            else:
+                value = getattr(record, field)
 
             if not value:
                 continue
@@ -481,6 +510,9 @@ def to_xlsx(project):
         if layers_data := docker.get_layers_data(project):
             _add_xlsx_worksheet(workbook, "LAYERS", layers_data, docker.layer_fields)
 
+        if todos_data := get_todos_data(project):
+            _add_xlsx_worksheet(workbook, "TODOS", todos_data, TODO_FIELDS)
+
     return output_file
 
 

scanpipe/tests/pipes/test_output.py

@@ -33,6 +33,7 @@
 from django.core.management import call_command
 from django.test import TestCase
 
+import openpyxl
 import xlsxwriter
 from licensedcode.cache import get_licensing
 from lxml import etree
@@ -42,10 +43,12 @@
 from scanpipe.models import CodebaseResource
 from scanpipe.models import Project
 from scanpipe.models import ProjectMessage
+from scanpipe.pipes import flag
 from scanpipe.pipes import output
 from scanpipe.tests import FIXTURES_REGEN
 from scanpipe.tests import make_dependency
 from scanpipe.tests import make_package
+from scanpipe.tests import make_resource_file
 from scanpipe.tests import mocked_now
 from scanpipe.tests import package_data1
 
@@ -210,16 +213,30 @@ def test_scanpipe_pipes_outputs_to_xlsx(self):
             model="Model",
             details={},
         )
+        make_resource_file(
+            project=project, path="path/file1.ext", status=flag.REQUIRES_REVIEW
+        )
 
         output_file = output.to_xlsx(project=project)
         self.assertIn(output_file.name, project.output_root)
 
         # Make sure the output can be generated even if the work_directory was wiped
         shutil.rmtree(project.work_directory)
-        with self.assertNumQueries(8):
+        with self.assertNumQueries(9):
             output_file = output.to_xlsx(project=project)
         self.assertIn(output_file.name, project.output_root)
 
+        workbook = openpyxl.load_workbook(output_file, read_only=True, data_only=True)
+        expected_sheet_names = [
+            "PACKAGES",
+            "DEPENDENCIES",
+            "RESOURCES",
+            "RELATIONS",
+            "MESSAGES",
+            "TODOS",
+        ]
+        self.assertEqual(expected_sheet_names, workbook.get_sheet_names())
+
     def test_scanpipe_pipes_outputs_vulnerability_as_cyclonedx(self):
         component_bom_ref = "pkg:pypi/[email protected]"
         data = self.data / "cyclonedx/django-4.0.10-vulnerability.json"
@@ -480,6 +497,34 @@ def test_scanpipe_pipes_outputs_to_attribution(self):
         output_file = output.to_attribution(project=project)
         self.assertEqual("EMPTY_TEMPLATE", output_file.read_text())
 
+    def test_scanpipe_pipes_outputs_get_todos_data(self):
+        project = Project.objects.create(name="Analysis")
+        todos_data = output.get_todos_data(project)
+        self.assertEqual([], list(todos_data))
+
+        make_resource_file(
+            project=project, path="path/file1.ext", status=flag.REQUIRES_REVIEW
+        )
+        make_resource_file(project=project, path="path/file2.ext")
+
+        todos_data = output.get_todos_data(project)
+        expected = [
+            {
+                "path": "path/file1.ext",
+                "status": "requires-review",
+                "size": None,
+                "name": "file1.ext",
+                "extension": ".ext",
+                "programming_language": "",
+                "mime_type": "",
+                "tag": "path",
+                "detected_license_expression": "",
+                "compliance_alert": "",
+                "project__name": "Analysis",
+            }
+        ]
+        self.assertEqual(expected, list(todos_data))
+
 
 class ScanPipeXLSXOutputPipesTest(TestCase):
     def test__add_xlsx_worksheet_does_truncates_long_strings_over_max_len(self):