Skip to content

Commit ec4c163

Browse files
committed
Add a report management command to generate XLSX reports #1524
Signed-off-by: tdruez <tdruez@nexb.com>
1 parent dd0e0bd commit ec4c163

File tree

5 files changed

+215
-0
lines changed

5 files changed

+215
-0
lines changed

CHANGELOG.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ v34.9.4 (unreleased)
4747
sheets with a dedicated VULNERABILITIES sheet.
4848
https://github.com/aboutcode-org/scancode.io/issues/1519
4949

50+
- Add a ``report`` management command that allows to generate XLSX reports for
51+
multiple projects at once using labels and searching by project name.
52+
https://github.com/aboutcode-org/scancode.io/issues/1524
53+
5054
v34.9.3 (2024-12-31)
5155
--------------------
5256

docs/command-line-interface.rst

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ ScanPipe's own commands are listed under the ``[scanpipe]`` section::
6868
list-project
6969
output
7070
purldb-scan-worker
71+
report
7172
reset-project
7273
run
7374
show-pipeline
@@ -393,6 +394,46 @@ your outputs on the host machine when running with Docker.
393394
.. tip:: To specify a CycloneDX spec version (default to latest), use the syntax
394395
``cyclonedx:VERSION`` as format value. For example: ``--format cyclonedx:1.5``.
395396

397+
.. _cli_report:
398+
399+
`$ scanpipe report --sheet SHEET`
400+
---------------------------------
401+
402+
Generates an XLSX report of selected projects based on the provided criteria.
403+
404+
Required arguments:
405+
406+
- ``--sheet {package,dependency,resource,relation,message,todo}``
407+
Specifies the sheet to include in the XLSX report. Available choices are based on
408+
predefined object types.
409+
410+
Optional arguments:
411+
412+
- ``--output-directory OUTPUT_DIRECTORY``
413+
The path to the directory where the report file will be created. If not provided,
414+
the report file will be created in the current working directory.
415+
416+
- ``--search SEARCH``
417+
Filter projects by searching for the provided string in their name.
418+
419+
- ``--label LABELS``
420+
Filter projects by the provided label(s). Multiple labels can be provided by using
421+
this argument multiple times.
422+
423+
.. note::
424+
Either ``--label`` or ``--search`` must be provided to select projects.
425+
426+
Example usage:
427+
428+
1. Generate a report for all projects tagged with "d2d" and include the **TODOS**
429+
worksheet::
430+
431+
$ scanpipe report --sheet todo --label d2d
432+
433+
2. Generate a report for projects whose names contain the word "audit" and include the
434+
**PACKAGES** worksheet::
435+
436+
$ scanpipe report --sheet package --search audit
396437

397438
.. _cli_check_compliance:
398439

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/aboutcode-org/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/aboutcode-org/scancode.io for support and download.
22+
23+
from pathlib import Path
24+
from timeit import default_timer as timer
25+
26+
from django.core.management import CommandError
27+
from django.core.management.base import BaseCommand
28+
29+
import xlsxwriter
30+
31+
from aboutcode.pipeline import humanize_time
32+
from scanpipe.models import Project
33+
from scanpipe.pipes import filename_now
34+
from scanpipe.pipes import output
35+
36+
37+
class Command(BaseCommand):
38+
help = "Report of selected projects."
39+
40+
def add_arguments(self, parser):
41+
super().add_arguments(parser)
42+
parser.add_argument(
43+
"--output-directory",
44+
help=(
45+
"The path to the directory where the report file will be created. "
46+
"If not provided, the report file will be created in the current "
47+
"working directory."
48+
),
49+
)
50+
parser.add_argument(
51+
"--sheet",
52+
required=True,
53+
choices=list(output.object_type_to_model_name.keys()),
54+
help="Specifies the sheet to include in the XLSX report.",
55+
)
56+
parser.add_argument(
57+
"--search",
58+
help="Select projects searching for the provided string in their name.",
59+
)
60+
parser.add_argument(
61+
"--label",
62+
action="append",
63+
dest="labels",
64+
default=list(),
65+
help=(
66+
"Filter projects by the provided label(s). Multiple labels can be "
67+
"provided by using this argument multiple times."
68+
),
69+
)
70+
71+
def handle(self, *args, **options):
72+
start_time = timer()
73+
self.verbosity = options["verbosity"]
74+
75+
output_directory = options["output_directory"]
76+
labels = options["labels"]
77+
search = options["search"]
78+
sheet = options["sheet"]
79+
model_name = output.object_type_to_model_name.get(sheet)
80+
81+
if not (labels or search):
82+
raise CommandError(
83+
"You must provide either --label or --search to select projects."
84+
)
85+
86+
project_qs = Project.objects.all()
87+
if labels:
88+
project_qs = project_qs.filter(labels__name__in=labels)
89+
if search:
90+
project_qs = project_qs.filter(name__icontains=search)
91+
project_count = project_qs.count()
92+
93+
if not project_count:
94+
raise CommandError("No projects found for the provided criteria.")
95+
96+
if self.verbosity > 0:
97+
msg = f"{project_count} project(s) will be included in the report."
98+
self.stdout.write(msg, self.style.SUCCESS)
99+
100+
worksheet_queryset = output.get_queryset(project=None, model_name=model_name)
101+
worksheet_queryset = worksheet_queryset.filter(project__in=project_qs)
102+
103+
filename = f"scancodeio-report-{filename_now()}.xlsx"
104+
if output_directory:
105+
output_file = Path(f"{output_directory}/{filename}")
106+
else:
107+
output_file = Path(filename)
108+
109+
with xlsxwriter.Workbook(output_file) as workbook:
110+
output.queryset_to_xlsx_worksheet(
111+
worksheet_queryset,
112+
workbook,
113+
prepend_fields=["project"],
114+
worksheet_name="TODOS",
115+
)
116+
117+
run_time = timer() - start_time
118+
if self.verbosity > 0:
119+
msg = f"Report generated at {output_file} in {humanize_time(run_time)}."
120+
self.stdout.write(msg, self.style.SUCCESS)

scanpipe/pipes/output.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,11 @@ def to_json(project):
309309
"codebaseresource": "resource",
310310
"codebaserelation": "relation",
311311
"projectmessage": "message",
312+
"todos": "todo",
313+
}
314+
315+
object_type_to_model_name = {
316+
value: key for key, value in model_name_to_object_type.items()
312317
}
313318

314319

scanpipe/tests/test_commands.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
import datetime
2424
import json
25+
import tempfile
2526
import uuid
2627
from contextlib import redirect_stdout
2728
from io import StringIO
@@ -37,14 +38,18 @@
3738
from django.test import override_settings
3839
from django.utils import timezone
3940

41+
import openpyxl
42+
4043
from scanpipe.management import commands
4144
from scanpipe.models import CodebaseResource
4245
from scanpipe.models import DiscoveredPackage
4346
from scanpipe.models import Project
4447
from scanpipe.models import Run
4548
from scanpipe.models import WebhookSubscription
49+
from scanpipe.pipes import flag
4650
from scanpipe.pipes import purldb
4751
from scanpipe.tests import make_package
52+
from scanpipe.tests import make_project
4853
from scanpipe.tests import make_resource_file
4954

5055
scanpipe_app = apps.get_app_config("scanpipe")
@@ -1092,6 +1097,46 @@ def test_scanpipe_management_command_check_compliance(self):
10921097
)
10931098
self.assertEqual(expected, out_value)
10941099

1100+
def test_scanpipe_management_command_report(self):
1101+
project1 = make_project("project1")
1102+
label1 = "label1"
1103+
project1.labels.add(label1)
1104+
make_resource_file(project1, path="file.ext", status=flag.REQUIRES_REVIEW)
1105+
make_project("project2")
1106+
1107+
expected = "Error: the following arguments are required: --sheet"
1108+
with self.assertRaisesMessage(CommandError, expected):
1109+
call_command("report")
1110+
1111+
options = ["--sheet", "UNKNOWN"]
1112+
expected = "Error: argument --sheet: invalid choice: 'UNKNOWN'"
1113+
with self.assertRaisesMessage(CommandError, expected):
1114+
call_command("report", *options)
1115+
1116+
options = ["--sheet", "todo"]
1117+
expected = "You must provide either --label or --search to select projects."
1118+
with self.assertRaisesMessage(CommandError, expected):
1119+
call_command("report", *options)
1120+
1121+
expected = "No projects found for the provided criteria."
1122+
with self.assertRaisesMessage(CommandError, expected):
1123+
call_command("report", *options, *["--label", "UNKNOWN"])
1124+
1125+
output_directory = Path(tempfile.mkdtemp())
1126+
options.extend(["--output-directory", str(output_directory), "--label", label1])
1127+
out = StringIO()
1128+
call_command("report", *options, stdout=out)
1129+
self.assertIn("1 project(s) will be included in the report.", out.getvalue())
1130+
output_file = list(output_directory.glob("*.xlsx"))[0]
1131+
self.assertIn(f"Report generated at {output_file}", out.getvalue())
1132+
1133+
workbook = openpyxl.load_workbook(output_file, read_only=True, data_only=True)
1134+
self.assertEqual(["TODOS"], workbook.get_sheet_names())
1135+
todos_sheet = workbook.get_sheet_by_name("TODOS")
1136+
row1 = list(todos_sheet.values)[1]
1137+
expected = ("project1", "file.ext", "file", "file.ext", "requires-review")
1138+
self.assertEqual(expected, row1[0:5])
1139+
10951140

10961141
class ScanPipeManagementCommandMixinTest(TestCase):
10971142
class CreateProjectCommand(

0 commit comments

Comments
 (0)