Address review comments

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

scanpipe/pipelines/__init__.py

@@ -182,6 +182,8 @@ def __init__(self, run_instance):
         self.selected_groups = run_instance.selected_groups
         self.selected_steps = run_instance.selected_steps
 
+        self.ecosystem_config = None
+
     @classmethod
     def get_initial_steps(cls):
         """Add the ``download_inputs`` step as an initial step if enabled."""

scanpipe/pipelines/deploy_to_develop.py

@@ -24,15 +24,15 @@
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import d2d
+from scanpipe.pipes import d2d_config
 from scanpipe.pipes import flag
 from scanpipe.pipes import input
 from scanpipe.pipes import matchcode
 from scanpipe.pipes import purldb
 from scanpipe.pipes import scancode
-from scanpipe.pipes.d2d_config import DefaultEcosystemConfig
 
 
-class DeployToDevelop(Pipeline, DefaultEcosystemConfig):
+class DeployToDevelop(Pipeline):
     """
     Establish relationships between two code trees: deployment and development.
 
@@ -133,7 +133,7 @@ def flag_whitespace_files(self):
 
     def load_ecosystem_config(self):
         """Load ecosystem specific configurations for d2d steps for selected options."""
-        d2d.load_ecosystem_config(pipeline=self, options=self.selected_groups)
+        d2d_config.load_ecosystem_config(pipeline=self, options=self.selected_groups)
 
     @optional_step("Ruby")
     def load_ecosystem_config_ruby(self):
@@ -285,12 +285,12 @@ def perform_house_keeping_tasks(self):
         d2d.handle_dangling_deployed_legal_files(project=self.project, logger=self.log)
         d2d.ignore_unmapped_resources_from_config(
             project=self.project,
-            patterns_to_ignore=self.deployed_resource_path_exclusions,
+            patterns_to_ignore=self.ecosystem_config.deployed_resource_path_exclusions,
             logger=self.log,
         )
         d2d.match_unmapped_resources(
             project=self.project,
-            matched_extensions=self.purldb_resource_extensions,
+            matched_extensions=self.ecosystem_config.purldb_resource_extensions,
             logger=self.log,
         )
         d2d.flag_undeployed_resources(project=self.project)
@@ -326,5 +326,5 @@ def flag_deployed_from_resources_with_missing_license(self):
         """Update the status for deployed from files with missing license."""
         d2d.flag_deployed_from_resources_with_missing_license(
             self.project,
-            doc_extensions=self.doc_extensions,
+            doc_extensions=self.ecosystem_config.doc_extensions,
         )

scanpipe/pipes/d2d.py

@@ -70,15 +70,6 @@
 TO = "to/"
 
 
-ECOSYSTEM_CONFIGS = [
-    d2d_config.DefaultEcosystemConfig,
-    d2d_config.JavaEcosystemConfig,
-    d2d_config.JavaScriptEcosystemConfig,
-    d2d_config.RubyEcosystemConfig,
-    d2d_config.RustEcosystemConfig,
-    d2d_config.GoEcosystemConfig,
-]
-
 
 def get_inputs(project):
     """
@@ -128,55 +119,6 @@ def get_best_path_matches(to_resource, matches):
     return matches
 
 
-def load_ecosystem_config(pipeline, options):
-    """
-    Add ecosystem specific configurations for each ecosystem selected
-    as `options` to the `pipeline`.
-    """
-    configs_by_ecosystem = {
-        ecosystem.ecosystem_option: ecosystem for ecosystem in ECOSYSTEM_CONFIGS
-    }
-
-    # Add default configurations which are common accross ecosystems
-    add_ecosystem_config(
-        pipeline=pipeline,
-        configs_by_ecosystem=configs_by_ecosystem,
-        selected_option="Default",
-    )
-
-    # Add configurations for each selected ecosystem
-    for selected_option in options:
-        if selected_option not in configs_by_ecosystem:
-            continue
-
-        add_ecosystem_config(
-            pipeline=pipeline,
-            configs_by_ecosystem=configs_by_ecosystem,
-            selected_option=selected_option,
-        )
-
-
-def add_ecosystem_config(pipeline, configs_by_ecosystem, selected_option):
-    d2d_pipeline_configs = [
-        "purldb_package_extensions",
-        "purldb_resource_extensions",
-        "deployed_resource_path_exclusions",
-    ]
-
-    ecosystem_config = configs_by_ecosystem.get(selected_option)
-
-    for pipeline_config in d2d_pipeline_configs:
-        config_value = getattr(ecosystem_config, pipeline_config)
-        pipeline_config_value = getattr(pipeline, pipeline_config)
-        if config_value:
-            if not pipeline_config_value:
-                new_config_value = config_value
-            else:
-                new_config_value = pipeline_config_value.extend(config_value)
-
-            setattr(pipeline, pipeline_config, new_config_value)
-
-
 def get_from_files_for_scanning(resources):
     """
     Return resources in the "from/" side which has been mapped to the "to/"

scanpipe/pipes/d2d_config.py

@@ -21,95 +21,155 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 
+from dataclasses import dataclass, field
+
+
+
+@dataclass
 class EcosystemConfig:
     """
-    Base class for ecosystem specific configurations to be defined
-    for each ecosystems.
+    Base class for ecosystem-specific configurations to be defined
+    for each ecosystem.
     """
 
     # This should be defined for each ecosystem which
     # are options in the pipelines
-    ecosystem_option = None
+    ecosystem_option: str = "Default"
 
     # These are extensions for packages of this ecosystem which
-    # needs to be matched from purldb
-    purldb_package_extensions = []
+    # need to be matched from purldb
+    purldb_package_extensions: list = field(default_factory=list)
 
     # These are extensions for resources of this ecosystem which
-    # needs to be macthed from purldb
-    purldb_resource_extensions = []
+    # need to be matched from purldb
+    purldb_resource_extensions: list = field(default_factory=list)
 
     # Extensions for document files which do not require review
-    doc_extensions = []
+    doc_extensions: list = field(default_factory=list)
 
     # Paths in the deployed binaries/archives (on the to/ side) which
     # do not need review even if they are not matched to the source side
-    deployed_resource_path_exclusions = []
+    deployed_resource_path_exclusions: list = field(default_factory=list)
 
-    # Paths in the developement/source archive (on the from/ side) which
+    # Paths in the development/source archive (on the from/ side) which
     # should not be considered even if unmapped to the deployed side when
-    # assesing what to review on the deployed side
-    devel_resource_path_exclusions = []
+    # assessing what to review on the deployed side
+    devel_resource_path_exclusions: list = field(default_factory=list)
 
-    # Symbols which are found in ecosystem specific standard libraries
+    # Symbols which are found in ecosystem-specific standard libraries
     # which are not so useful in mapping
-    standard_symbols_to_exclude = []
-
-
-class DefaultEcosystemConfig(EcosystemConfig):
-    """Configurations which are common across multiple ecosystems."""
-
-    ecosystem_option = "Default"
-    purldb_package_extensions = [".zip", ".tar.gz", ".tar.xz"]
-    devel_resource_path_exclusions = ["*/tests/*"]
-    doc_extensions = [
-        ".pdf",
-        ".doc",
-        ".docx",
-        ".ppt",
-        ".pptx",
-        ".tex",
-        ".odt",
-        ".odp",
-    ]
+    standard_symbols_to_exclude: list = field(default_factory=list)
+
+
+# Dictionary of ecosystem configurations
+ECOSYSTEM_CONFIGS = {
+    "Default": EcosystemConfig(
+        purldb_package_extensions=[".zip", ".tar.gz", ".tar.xz"],
+        devel_resource_path_exclusions=["*/tests/*"],
+        doc_extensions=[
+            ".pdf",
+            ".doc",
+            ".docx",
+            ".ppt",
+            ".pptx",
+            ".tex",
+            ".odt",
+            ".odp",
+        ],
+    ),
+    "Java": EcosystemConfig(
+        ecosystem_option="Java",
+        purldb_package_extensions=[".jar", ".war"],
+        purldb_resource_extensions=[".class"],
+    ),
+    "JavaScript": EcosystemConfig(
+        ecosystem_option="JavaScript",
+        purldb_resource_extensions=[
+            ".map",
+            ".js",
+            ".mjs",
+            ".ts",
+            ".d.ts",
+            ".jsx",
+            ".tsx",
+            ".css",
+            ".scss",
+            ".less",
+            ".sass",
+            ".soy",
+        ],
+    ),
+    "Go": EcosystemConfig(
+        ecosystem_option="Go",
+        purldb_resource_extensions=[".go"],
+    ),
+    "Rust": EcosystemConfig(
+        ecosystem_option="Rust",
+        purldb_resource_extensions=[".rs"],
+    ),
+    "Ruby": EcosystemConfig(
+        ecosystem_option="Ruby",
+        purldb_package_extensions=[".gem"],
+        purldb_resource_extensions=[".rb"],
+        deployed_resource_path_exclusions=["*checksums.yaml.gz*", "*metadata.gz*"],
+    ),
+}
+
+
+def get_ecosystem_config(ecosystem):
+    """Return the ``ecosystem`` config."""
+    return ECOSYSTEM_CONFIGS.get(ecosystem, ECOSYSTEM_CONFIGS["Default"])
+
+
+def load_ecosystem_config(pipeline, options):
+    """
+    Add ecosystem specific configurations for each ecosystem selected
+    as `options` to the `pipeline`. These configurations are used for:
+    - which resource/package extensions to match to purldb
+    - which source files to get source symbols from
+    - which unmapped paths to ignore in deployed binaries 
+    """
+    configs_by_ecosystem = {
+        ecosystem.ecosystem_option: ecosystem for ecosystem in ECOSYSTEM_CONFIGS.values()
+    }
 
+    # Add default configurations which are common accross ecosystems
+    pipeline.ecosystem_config = ECOSYSTEM_CONFIGS.get("Default")
 
-class JavaEcosystemConfig(EcosystemConfig):
-    ecosystem_option = "Java"
-    purldb_package_extensions = [".jar", ".war"]
-    purldb_resource_extensions = [".class"]
-
-
-class JavaScriptEcosystemConfig(EcosystemConfig):
-    ecosystem_option = "JavaScript"
-    purldb_resource_extensions = [
-        ".map",
-        ".js",
-        ".mjs",
-        ".ts",
-        ".d.ts",
-        ".jsx",
-        ".tsx",
-        ".css",
-        ".scss",
-        ".less",
-        ".sass",
-        ".soy",
-    ]
+    # Add configurations for each selected ecosystem
+    for selected_option in options:
+        if selected_option not in configs_by_ecosystem:
+            continue
+
+        add_ecosystem_config(
+            pipeline_ecosystem_config=pipeline.ecosystem_config,
+            configs_by_ecosystem=configs_by_ecosystem,
+            selected_option=selected_option,
+        )
 
 
-class GoEcosystemConfig(EcosystemConfig):
-    ecosystem_option = "Go"
-    purldb_resource_extensions = [".go"]
+def add_ecosystem_config(pipeline_ecosystem_config, configs_by_ecosystem, selected_option):
+    """
+    Set the `pipeline_ecosystem_config` values from all the individual ecosystem based configurations
+    defined in `configs_by_ecosystem`, based on pipeline `selected_option` which selects an
+    ecosytem.
+    """
+    d2d_pipeline_configs = [
+        "purldb_package_extensions",
+        "purldb_resource_extensions",
+        "deployed_resource_path_exclusions",
+    ]
 
+    ecosystem_config = configs_by_ecosystem.get(selected_option)
 
-class RustEcosystemConfig(EcosystemConfig):
-    ecosystem_option = "Rust"
-    purldb_resource_extensions = [".rs"]
+    for config_name in d2d_pipeline_configs:
+        config_value = getattr(ecosystem_config, config_name)
+        pipeline_config_value = getattr(pipeline_ecosystem_config, config_name)
+        if config_value:
+            if not pipeline_config_value:
+                new_config_value = config_value
+            else:
+                new_config_value = pipeline_config_value.extend(config_value)
 
+            setattr(pipeline_ecosystem_config, config_name, new_config_value)
 
-class RubyEcosystemConfig(EcosystemConfig):
-    ecosystem_option = "Ruby"
-    purldb_package_extensions = [".gem"]
-    purldb_resource_extensions = [".rb"]
-    deployed_resource_path_exclusions = ["*checksums.yaml.gz*", "*metadata.gz*"]