feat: add pipeline specialized in Maven package scan (#1953)

Signed-off-by: Chin Yeung Li <tli@nexb.com>

Author: chinyeungli

docs/built-in-pipelines.rst

@@ -265,6 +265,14 @@ Scan Single Package
     :members:
     :member-order: bysource
 
+.. _pipeline_scan_maven_package:
+
+Scan Maven Package
+-------------------
+.. autoclass:: scanpipe.pipelines.scan_maven_package.ScanMavenPackage()
+    :members:
+    :member-order: bysource
+
 Fetch Scores (addon)
 --------------------
 .. warning::

pyproject.toml

@@ -170,6 +170,7 @@ publish_to_federatedcode = "scanpipe.pipelines.publish_to_federatedcode:PublishT
 resolve_dependencies = "scanpipe.pipelines.resolve_dependencies:ResolveDependencies"
 scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase"
 scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus"
+scan_maven_package = "scanpipe.pipelines.scan_maven_package:ScanMavenPackage"
 scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage"
 
 [tool.setuptools.packages.find]

scanpipe/pipelines/scan_maven_package.py

@@ -0,0 +1,185 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from scanpipe.pipelines.deploy_to_develop import DeployToDevelop
+from scanpipe.pipelines.scan_single_package import ScanSinglePackage
+from scanpipe.pipes import d2d_config
+from scanpipe.pipes.maven import check_input_and_return_purl
+from scanpipe.pipes.maven import fetch_and_scan_remote_pom
+from scanpipe.pipes.maven import fetch_inputs
+from scanpipe.pipes.maven import update_package_license_from_resource_if_missing
+
+
+class ScanMavenPackage(ScanSinglePackage, DeployToDevelop):
+    """
+    Scan a single Maven package and perform a deployment to development
+    relation scan.
+
+    This pipeline takes a Maven PURL as input, fetches the binary and
+    source archives (if they exist), and then performs scans for package
+    metadata, declared dependencies, licenses, license clarity scores, and
+    copyrights. It also performs a deployment to development relation scan
+    if both the source and binary archives are available.
+
+    The output is a summary of the scan results in JSON format.
+    """
+
+    download_inputs = False
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_input_and_return_purl,
+            cls.fetch_inputs,
+            cls.d2d_check,
+            cls.collect_input_info,
+            cls.extract_input,
+            cls.extract_archives,
+            cls.maven_d2d_steps,
+            cls.run_scan,
+            cls.fetch_and_scan_remote_pom,
+            cls.load_inventory_from_toolkit_scan,
+            cls.update_package_license_from_resource_if_missing,
+            cls.make_summary_from_scan_results,
+        )
+
+    def check_input_and_return_purl(self):
+        """Validate the input is a PURL string and return the PURL object."""
+        self.purl = check_input_and_return_purl(self.project)
+
+    def fetch_inputs(self):
+        """Fetch the binary and source of the given PURL."""
+        from_file, to_file = fetch_inputs(self.purl)
+        # The pipeline will call self.extract_inputs_to_codebase_directory()
+        # from deploy_to_develop.py, which expects both self.from_files and
+        # self.to_files to be of list type.
+        self.from_files = [from_file] if from_file else []
+        self.to_files = [to_file] if to_file else []
+
+    def d2d_check(self):
+        """
+        Enable the deployment to development relation scan if both from
+        and to files are present.
+        """
+        self.d2d_enabled = False
+        if self.from_files and self.to_files:
+            self.d2d_enabled = True
+
+    def collect_input_info(self):
+        """Collect information about the input."""
+        if not self.d2d_enabled:
+            if self.from_files:
+                self.input_path = self.from_files[0]
+            else:
+                self.input_path = self.to_files[0]
+            self.collect_input_information()
+
+    def extract_input(self):
+        """Extract the input to the codebase directory."""
+        if not self.d2d_enabled:
+            self.extract_input_to_codebase_directory()
+        else:
+            self.extract_inputs_to_codebase_directory()
+
+    def maven_d2d_steps(self):
+        """Run deployment to development relation scan steps for Maven projects."""
+        if self.d2d_enabled:
+            self.collect_and_create_codebase_resources()
+            self.fingerprint_codebase_directories()
+            self.flag_empty_files()
+            self.flag_whitespace_files()
+            self.flag_ignored_resources()
+            self.d2d_ecosystem_config()
+            self.map_about_files()
+            self.map_checksum()
+            self.match_archives_to_purldb()
+            self.d2d_java()
+            self.d2d_scala()
+            self.d2d_kotlin()
+            self.d2d_javascript()
+            self.d2d_process()
+
+    def d2d_ecosystem_config(self):
+        options = ["Java", "Kotlin", "Scala", "JavaScript"]
+        d2d_config.load_ecosystem_config(pipeline=self, options=options)
+
+    def d2d_java(self):
+        self.find_java_packages()
+        self.map_java_to_class()
+        self.map_jar_to_java_source()
+
+    def d2d_scala(self):
+        self.find_scala_packages()
+        self.map_scala_to_class()
+        self.map_jar_to_scala_source()
+
+    def d2d_kotlin(self):
+        self.find_kotlin_packages()
+        self.map_kotlin_to_class()
+        self.map_jar_to_kotlin_source()
+
+    def d2d_javascript(self):
+        self.map_javascript()
+        self.map_javascript_symbols()
+        self.map_javascript_strings()
+
+    def d2d_process(self):
+        self.get_symbols_from_binaries()
+        self.map_elf()
+        self.match_directories_to_purldb()
+        self.match_resources_to_purldb()
+        self.map_javascript_post_purldb_match()
+        self.map_javascript_path()
+        self.map_javascript_colocation()
+        self.map_path()
+        self.flag_mapped_resources_archives_and_ignored_directories()
+        self.perform_house_keeping_tasks()
+        self.match_purldb_resources_post_process()
+        self.remove_packages_without_resources()
+        self.scan_ignored_to_files()
+        self.scan_unmapped_to_files()
+        self.scan_mapped_from_for_files()
+        self.collect_and_create_license_detections()
+        self.flag_deployed_from_resources_with_missing_license()
+        self.create_local_files_packages()
+
+    def fetch_and_scan_remote_pom(self):
+        """Fetch and scan remote POM files."""
+        scanning_errors = fetch_and_scan_remote_pom(
+            self.purl, self.scan_output_location
+        )
+        for scanning_error in scanning_errors:
+            for resource_reference, errors in scanning_error.items():
+                self.project.add_error(
+                    description="\n".join(errors),
+                    model=self.pipeline_name,
+                    details={
+                        "resource_path": resource_reference.removeprefix("codebase/")
+                    },
+                )
+
+    def update_package_license_from_resource_if_missing(self):
+        """
+        Fill in missing package licenses using licenses detected in
+        their resources.
+        """
+        update_package_license_from_resource_if_missing(self.project)

scanpipe/pipes/d2d.py

@@ -353,7 +353,7 @@ def map_jar_to_jvm_source(project, jvm_lang: jvm.JvmLanguage, logger=None):
     """Map .jar files to their related source directory."""
     project_files = project.codebaseresources.files()
     # Include the directories to map on the common source
-    from_resources = project.codebaseresources.from_codebase()
+    from_resources = project.codebaseresources.from_codebase().has_no_relation()
     to_resources = project_files.to_codebase()
     to_jars = to_resources.filter(extension=".jar")