Merge branch 'main' into exclusion-framework-ruby

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

CHANGELOG.rst

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+v35.1.0 (unreleased)
+--------------------
+
+- Add a ``--fail-on-vulnerabilities`` option in ``check-compliance`` management command.
+  When this option is enabled, the command will exit with a non-zero status if known
+  vulnerabilities are detected in discovered packages and dependencies.
+  Requires the ``find_vulnerabilities`` pipeline to be executed beforehand.
+  https://github.com/aboutcode-org/scancode.io/pull/1702
+
 v35.0.0 (2025-06-23)
 --------------------
 
@@ -36,6 +45,11 @@ v35.0.0 (2025-06-23)
 - Add "Package Compliance Alert" chart in the Policies section.
   https://github.com/aboutcode-org/scancode.io/pull/1699
 
+- Update univers to v31.0.0, catch ``NotImplementedError`` in
+  ``get_unique_unresolved_purls``, and properly log error in project.
+  https://github.com/aboutcode-org/scancode.io/pull/1700
+  https://github.com/aboutcode-org/scancode.io/pull/1701
+
 v34.11.0 (2025-05-02)
 ---------------------
 

docs/command-line-interface.rst

@@ -497,6 +497,10 @@ Optional arguments:
 - ``--fail-level {ERROR,WARNING,MISSING}`` Compliance alert level that will cause the
   command to exit with a non-zero status. Default is ERROR.
 
+- ``--fail-on-vulnerabilities`` Exit with a non-zero status if known vulnerabilities
+  are detected in discovered packages and dependencies.
+  Requires the ``find_vulnerabilities`` pipeline to be executed beforehand.
+
 `$ scanpipe archive-project --project PROJECT`
 ----------------------------------------------
 

scancodeio/static/add-inputs.js

@@ -21,15 +21,41 @@
 // Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 const fileInput = document.querySelector("#id_input_files");
+let selectedFiles = []; // Store selected files
 fileInput.onchange = updateFiles;
 
 // Update the list of files to be uploaded in the UI
 function updateFiles() {
   if (fileInput.files.length > 0) {
     const fileName = document.querySelector("#inputs_file_name");
     fileName.innerHTML = "";
-    for (let file of fileInput.files) {
-      fileName.innerHTML += `<span class="is-block">${file.name}</span>`;
+
+    // Update the selectedFiles array
+    const newFiles = Array.from(fileInput.files);
+    // Create a Set to track unique file names
+    const uniqueFileNames = new Set(selectedFiles.map(file => file.name));
+    // Filter out files with the same name
+    const filteredNewFiles = newFiles.filter(file => !uniqueFileNames.has(file.name));
+    // Concatenate the unique files to the existing selectedFiles array
+    selectedFiles = selectedFiles.concat(filteredNewFiles);
+
+    for (let file of selectedFiles) {
+      const fileNameWithoutSpaces = file.name.replace(/\s/g, '');
+      fileName.innerHTML += `
+      <span class="is-flex is-justify-content-space-between is-block" id="file-name-${fileNameWithoutSpaces}">
+        <span class="is-block">${file.name}</span>
+        <a href="#" onclick="removeFile('${fileNameWithoutSpaces}')" class="model-button" id="file-delete-btn-${fileNameWithoutSpaces}">
+          <i class="fa-solid fa-trash-can"></i>
+        </a>
+      </span>
+      `;
+      document.getElementById("file-delete-btn-"+ fileNameWithoutSpaces).addEventListener("click", function(event){
+        disableEvent(event);
+        removeFile(fileNameWithoutSpaces);
+        if(selectedFiles.length == 0){
+          fileName.innerHTML ="<i>No files selected</i>"
+        }
+      });
     }
   }
 }
@@ -40,15 +66,37 @@ function disableEvent(event) {
   event.preventDefault();
 }
 
+function removeFile(fileName) {
+  selectedFiles = selectedFiles.filter(file => {
+    const fileNameWithoutSpaces = file.name.replace(/\s/g, '');
+    return fileNameWithoutSpaces !== fileName;
+  });
+
+  const fileNameElement = document.getElementById(`file-name-${fileName}`);
+  if (fileNameElement) {
+    fileNameElement.remove();
+  }
+
+  const dataTransfer = new DataTransfer();
+  for (let file of selectedFiles) {
+    dataTransfer.items.add(file);
+  }
+
+  fileInput.files = dataTransfer.files;
+}
+
 function dropHandler(event) {
   disableEvent(event);
   const droppedFiles = event.dataTransfer.files;
-  const updatedFiles = Array.from(fileInput.files);
+  const updatedFilesSet = new Set(Array.from(fileInput.files));
 
   for (let file of droppedFiles) {
-    updatedFiles.push(file);
+    updatedFilesSet.add(file);
   }
 
+  // Convert the Set back to an array if needed
+  const updatedFiles = Array.from(updatedFilesSet);
+
   const dataTransfer = new DataTransfer();
   for (let file of updatedFiles) {
     dataTransfer.items.add(file);

scanpipe/management/commands/check-compliance.py

@@ -45,31 +45,64 @@ def add_arguments(self, parser):
                 "non-zero status. Default is ERROR."
             ),
         )
+        parser.add_argument(
+            "--fail-on-vulnerabilities",
+            action="store_true",
+            help=(
+                "Exit with a non-zero status if known vulnerabilities are detected in "
+                "discovered packages and dependencies. "
+                "Requires the `find_vulnerabilities` pipeline to be executed "
+                "beforehand."
+            ),
+        )
 
     def handle(self, *args, **options):
         super().handle(*args, **options)
-        fail_level = options["fail_level"]
-        compliance_alerts = get_project_compliance_alerts(self.project, fail_level)
+        exit_code = 0
+
+        if self.check_compliance(options["fail_level"]):
+            exit_code = 1
+
+        if options["fail_on_vulnerabilities"] and self.check_vulnerabilities():
+            exit_code = 1
 
-        compliance_alerts_count = sum(
-            len(issues_by_severity)
-            for model_alerts in compliance_alerts.values()
-            for issues_by_severity in model_alerts.values()
+        sys.exit(exit_code)
+
+    def check_compliance(self, fail_level):
+        alerts = get_project_compliance_alerts(self.project, fail_level)
+        count = sum(
+            len(issues) for model in alerts.values() for issues in model.values()
         )
-        if not compliance_alerts_count:
-            sys.exit(0)
 
-        if self.verbosity > 0:
-            msg = [
-                f"{compliance_alerts_count} compliance issues detected on this project."
-            ]
-            for label, issues in compliance_alerts.items():
-                msg.append(f"[{label}]")
-                for severity, entries in issues.items():
-                    msg.append(f" > {severity.upper()}: {len(entries)}")
+        if count and self.verbosity > 0:
+            self.stderr.write(f"{count} compliance issues detected.")
+            for label, model in alerts.items():
+                self.stderr.write(f"[{label}]")
+                for severity, entries in model.items():
+                    self.stderr.write(f" > {severity.upper()}: {len(entries)}")
                     if self.verbosity > 1:
-                        msg.append("   " + "\n   ".join(entries))
+                        self.stderr.write("   " + "\n   ".join(entries))
+
+        return count > 0
 
-            self.stderr.write("\n".join(msg))
+    def check_vulnerabilities(self):
+        packages = self.project.discoveredpackages.vulnerable_ordered()
+        dependencies = self.project.discovereddependencies.vulnerable_ordered()
+
+        vulnerable_records = list(packages) + list(dependencies)
+        count = len(vulnerable_records)
+
+        if self.verbosity > 0:
+            if count:
+                self.stderr.write(f"{count} vulnerable records found:")
+                for entry in vulnerable_records:
+                    self.stderr.write(str(entry))
+                    vulnerability_ids = [
+                        vulnerability.get("vulnerability_id")
+                        for vulnerability in entry.affected_by_vulnerabilities
+                    ]
+                    self.stderr.write(" > " + ", ".join(vulnerability_ids))
+            else:
+                self.stdout.write("No vulnerabilities found")
 
-        sys.exit(1)
+        return count > 0

scanpipe/models.py

@@ -3150,6 +3150,13 @@ class VulnerabilityQuerySetMixin:
     def vulnerable(self):
         return self.filter(~Q(affected_by_vulnerabilities__in=EMPTY_VALUES))
 
+    def vulnerable_ordered(self):
+        return (
+            self.vulnerable()
+            .only_package_url_fields(extra=["affected_by_vulnerabilities"])
+            .order_by_package_url()
+        )
+
 
 class DiscoveredPackageQuerySet(
     VulnerabilityQuerySetMixin,

scanpipe/pipes/d2d.py

@@ -54,7 +54,6 @@
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import convert_glob_to_django_regex
-from scanpipe.pipes import d2d_config
 from scanpipe.pipes import flag
 from scanpipe.pipes import get_resource_diff_ratio
 from scanpipe.pipes import js
@@ -70,7 +69,6 @@
 TO = "to/"
 
 
-
 def get_inputs(project):
     """
     Locate the ``from`` and ``to`` input files in project inputs/ directory.

scanpipe/pipes/d2d_config.py

@@ -21,8 +21,8 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 
-from dataclasses import dataclass, field
-
+from dataclasses import dataclass
+from dataclasses import field
 
 
 @dataclass
@@ -127,10 +127,11 @@ def load_ecosystem_config(pipeline, options):
     as `options` to the `pipeline`. These configurations are used for:
     - which resource/package extensions to match to purldb
     - which source files to get source symbols from
-    - which unmapped paths to ignore in deployed binaries 
+    - which unmapped paths to ignore in deployed binaries
     """
     configs_by_ecosystem = {
-        ecosystem.ecosystem_option: ecosystem for ecosystem in ECOSYSTEM_CONFIGS.values()
+        ecosystem.ecosystem_option: ecosystem
+        for ecosystem in ECOSYSTEM_CONFIGS.values()
     }
 
     # Add default configurations which are common accross ecosystems
@@ -148,11 +149,13 @@ def load_ecosystem_config(pipeline, options):
         )
 
 
-def add_ecosystem_config(pipeline_ecosystem_config, configs_by_ecosystem, selected_option):
+def add_ecosystem_config(
+    pipeline_ecosystem_config, configs_by_ecosystem, selected_option
+):
     """
-    Set the `pipeline_ecosystem_config` values from all the individual ecosystem based configurations
-    defined in `configs_by_ecosystem`, based on pipeline `selected_option` which selects an
-    ecosytem.
+    Set the `pipeline_ecosystem_config` values from all the individual ecosystem
+    based configurations defined in `configs_by_ecosystem`, based on pipeline
+    `selected_option` which selects an ecosytem.
     """
     d2d_pipeline_configs = [
         "purldb_package_extensions",
@@ -172,4 +175,3 @@ def add_ecosystem_config(pipeline_ecosystem_config, configs_by_ecosystem, select
                 new_config_value = pipeline_config_value.extend(config_value)
 
             setattr(pipeline_ecosystem_config, config_name, new_config_value)
-