feat: java transitive search - added support for project pom aggregator

Signed-off-by: Theodor Mihalache <[email protected]>

Author: tmihalac

kustomize/base/exploit_iq_service.yaml

@@ -123,9 +123,13 @@ spec:
                   fieldPath: metadata.namespace
             - name: GOMODCACHE
               value: /exploit-iq-package-cache/go/pkg/mod
+            - name: JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH
+              value: /maven-config/settings.xml
           volumeMounts:
             - name: config
               mountPath: /configs
+            - name: maven-settings-config
+              mountPath: /maven-config
             - name: cache
               mountPath: /exploit-iq-data
             - name: package-cache
@@ -137,6 +141,9 @@ spec:
         - name: config
           configMap:
             name: exploit-iq-config
+        - name: maven-settings-config
+          configMap:
+            name: exploit-iq-maven-settings-config
         - name: cache
           persistentVolumeClaim:
             claimName: exploit-iq-data

kustomize/base/kustomization.yaml

@@ -61,6 +61,9 @@ configMapGenerator:
     files:
       - excludes.json
       - includes.json
+  - name: exploit-iq-maven-settings-config
+    files:
+      - settings.xml
 
 patches:
   - path: ips-patch.json

kustomize/base/settings.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.2.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.2.0
+                              https://maven.apache.org/xsd/settings-1.2.0.xsd">
+    <profiles>
+        <profile>
+            <id>red-hat</id>
+            <repositories>
+                <repository>
+                    <id>red-hat-ga</id>
+                    <url>https://maven.repository.redhat.com/ga</url>
+                </repository>
+            </repositories>
+            <pluginRepositories>
+                <pluginRepository>
+                    <id>red-hat-ga</id>
+                    <url>https://maven.repository.redhat.com/ga</url>
+                    <releases>
+                        <enabled>true</enabled>
+                    </releases>
+                    <snapshots>
+                        <enabled>false</enabled>
+                    </snapshots>
+                </pluginRepository>
+            </pluginRepositories>
+        </profile>
+    </profiles>
+    <activeProfiles>
+        <activeProfile>red-hat</activeProfile>
+    </activeProfiles>
+
+    <mirrors>
+        <!-- Override Maven's built-in "maven-default-http-blocker" by reusing the same id,
+             but making it not match external:http:* anymore. -->
+        <mirror>
+            <id>maven-default-http-blocker</id>
+            <name>Disabled default HTTP blocker</name>
+
+            <!-- Make this mirror effectively never apply -->
+            <mirrorOf>__no_such_repo_id__</mirrorOf>
+
+            <!-- URL is irrelevant if mirrorOf never matches, but must be present -->
+            <url>https://repo1.maven.org/maven2</url>
+        </mirror>
+    </mirrors>
+
+</settings>

src/exploit_iq_commons/utils/dep_tree.py

@@ -841,9 +841,10 @@ def __init__(self, query: str):
         self._query = query
 
     def install_dependencies(self, manifest_path: Path):
+        settings_path = os.getenv('JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH','../../../../kustomize/base/settings.xml')
         source_path = "dependencies-sources"
-        subprocess.run(["mvn", "dependency:copy-dependencies", "-Dclassifier=sources",
-                        f"-DoutputDirectory={source_path}"], cwd=manifest_path)
+        subprocess.run(["mvn", "-s", settings_path, "dependency:copy-dependencies", "-Dclassifier=sources",
+                        "-DincludeScope=runtime", f"-DoutputDirectory={manifest_path.resolve()}/{source_path}"], cwd=manifest_path)
 
         full_source_path =  manifest_path / source_path
         for jar in full_source_path.glob("*-sources.jar"):
@@ -855,25 +856,59 @@ def install_dependencies(self, manifest_path: Path):
                     zf.extractall(dest)
 
     def build_tree(self, manifest_path: Path) -> dict[str, list[str]]:
+        settings_path = os.getenv("JAVA_MAVEN_DEFAULT_SETTINGS_FILE_PATH", "../../../../kustomize/base/settings.xml")
         dependency_file = manifest_path / "dependency_tree.txt"
-        package_name = self._query.split(',')[0]
+        package_name = self._query.split(",")[0]
 
         if is_maven_gav(package_name):
-            with open(dependency_file, "w") as f:
-                subprocess.run(["mvn", "dependency:tree",
-                                f"-Dincludes={add_missing_jar_string(package_name)}",
-                                "-Dverbose"], cwd=manifest_path, stdout=f, check=True)
+            subprocess.run(
+                [
+                    "mvn",
+                    "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate",
+                    "-s", settings_path,
+                    "-DgraphFormat=text",
+                    "-DshowGroupIds",
+                    "-DshowVersions",
+                    "-DshowTypes",
+                    "-DoutputDirectory=.",
+                    "-DoutputFileName=dependency_tree.txt",
+                    "-DclasspathScope=runtime",
+                    f"-DtargetIncludes={add_missing_jar_string(package_name)}",
+                ],
+                cwd=manifest_path,
+                check=True,
+            )
         else:
-            with open(dependency_file, "w") as f:
-                subprocess.run(["mvn", "dependency:tree",
-                                "-Dverbose"], cwd=manifest_path, stdout=f, check=True)
+            subprocess.run(
+                [
+                    "mvn",
+                    "com.github.ferstl:depgraph-maven-plugin:4.0.3:aggregate",
+                    "-s", settings_path,
+                    "-DgraphFormat=text",
+                    "-DshowGroupIds",
+                    "-DshowVersions",
+                    "-DshowTypes",
+                    "-DoutputDirectory=.",
+                    "-DclasspathScope=runtime",
+                    "-DoutputFileName=dependency_tree.txt",
+                ],
+                cwd=manifest_path,
+                check=True,
+            )
 
         with dependency_file.open("r", encoding="utf-8") as f:
             lines = f.readlines()
 
         parent, graph = self.__build_upside_down_dependency_graph(lines)
 
-        # Mark the top level for
+        # Mark *all* roots, not just the first one.
+        # A "root" is any node with no parents in the computed parent-chain list.
+        # This preserves old single-root behavior and fixes multi-root / multi-parent trees.
+        roots = [node for node, parents in graph.items() if not parents]
+        for r in roots:
+            graph[r] = [ROOT_LEVEL_SENTINEL]
+
+        # Backward-compatible: keep the old behavior too (harmless if already set above)
         graph[parent] = [ROOT_LEVEL_SENTINEL]
 
         return graph
@@ -883,16 +918,16 @@ def __build_upside_down_dependency_graph(
     ) -> Tuple[str, Dict[str, List[str]]]:
         root: str = ""
         stack: List[str] = []
-        # coord -> set of direct parents (possibly multiple)
-        graph_sets: Dict[str, set] = {}
+        graph_sets: Dict[str, set[str]] = {}  # coord -> set of direct parents
 
         for line in dependency_lines:
             depth, coord = self.__parse_dependency_line(line)
             if depth is None or coord is None:
                 continue
 
             if depth == 0:
-                # start (or restart) a root line
+                # depgraph aggregate can emit multiple top-level roots. Keep the first as "root"
+                # for backward compatibility, but still record others as separate roots in graph_sets.
                 if not root:
                     root = coord
                 stack = [coord]
@@ -905,91 +940,128 @@ def __build_upside_down_dependency_graph(
 
             parent = stack[-1] if stack else None
             if parent is not None:
-                graph_sets.setdefault(coord, set()).add(parent)
+                graph_sets.setdefault(coord, set()).add(parent)  # supports multiple direct parents
                 graph_sets.setdefault(parent, set())
             else:
                 graph_sets.setdefault(coord, set())
 
             stack.append(coord)
 
-        # ---------- second phase: all parents (direct + transitive) without duplicates ----------
-
         def build_parent_chain(node: str) -> List[str]:
             """
-            For a given coord, return a flat list of *all* parents reachable
-            via any path up to the root, with no duplicates.
-
-            Order: breadth-first from nearest parents outward.
+            Return a flat list of all parents reachable via any path, no duplicates.
+            Deterministic BFS order: nearest parents outward.
             """
             result: List[str] = []
             seen: set[str] = set()
 
-            q = deque(graph_sets.get(node, ()))
+            q = deque(sorted(graph_sets.get(node, ())))
             while q:
-                parent = q.popleft()
-                if parent in seen:
+                p = q.popleft()
+                if p in seen:
                     continue
-                seen.add(parent)
-                result.append(parent)
+                seen.add(p)
+                result.append(p)
 
-                # enqueue this parent's parents
-                for gp in graph_sets.get(parent, ()):
+                for gp in sorted(graph_sets.get(p, ())):
                     if gp not in seen:
                         q.append(gp)
 
             return result
 
-        graph: Dict[str, List[str]] = {
-            coord: build_parent_chain(coord) for coord in graph_sets.keys()
-        }
-
+        graph: Dict[str, List[str]] = {coord: build_parent_chain(coord) for coord in graph_sets.keys()}
         return root, graph
 
     def __parse_dependency_line(self, line: str) -> Tuple[Optional[int], Optional[str]]:
-        if not line.startswith("[INFO]"):
+        """
+        Parse one dependency line from depgraph's graphFormat=text output.
+
+        Expected depgraph token shape (after indentation/branch prefix):
+          groupId:artifactId:version:type:scope
+        Example from your file:
+          org.apache.activemq:artemis-openwire-protocol:2.28.0:bundle:compile  :contentReference[oaicite:3]{index=3}
+
+        We return (depth, "groupId:artifactId:version") and ignore type/scope/optional marker.
+
+        Also tolerates Maven log prefixes like "[INFO] " if they appear.
+        """
+        raw = (line or "").rstrip("\n")
+        if not raw.strip():
+            return None, None
+
+        # If Maven stdout and depgraph output got mixed, you may see mid-line "[INFO]" injection.
+        # Those lines are not safely recoverable as dependency tokens.
+        if "[INFO]" in raw and not raw.lstrip().startswith("[INFO]"):
             return None, None
 
-        # Keep indentation blocks; Maven prints exactly one space after "[INFO]"
-        s = line[6:]
-        if s.startswith(" "):
-            s = s[1:]
+        s = raw.lstrip()
 
-        # Skip non-tree lines early
-        if (not s
-                or s.startswith(("---", "BUILD", "Scanning", "Finished", "Total time"))
-                or ":" not in s):
+        # Strip Maven log prefix if present
+        if s.startswith("[INFO]"):
+            s = s[6:].lstrip()
+
+        # Skip headers and build noise
+        if (
+                not s
+                or "Dependency graph:" in s
+                or s.startswith(("---", "BUILD", "Reactor Summary", "Total time", "Finished at", "Scanning"))
+                or s.startswith("[")  # other log levels like [WARNING], [ERROR], etc.
+                or ":" not in s
+        ):
             return None, None
 
-        # indent blocks ("|  " or "   ") + optional "+- " or "\- " + rest
-        m = re.match(r'^(?P<indent>(?:\|  |   )*)(?P<branch>[+\\]-\s)?(?P<rest>.+)$', s)
+        # depgraph indentation blocks ("|  " or "   ") + optional "+- " or "\- " + rest
+        m = re.match(r"^(?P<indent>(?:\|  |   )*)(?P<branch>[+\\]-\s)?(?P<rest>.+)$", s)
         if not m:
             return None, None
 
-        # Each indent block is 3 chars; add 1 if a branch token is present
-        depth = (len(m.group('indent')) // 3) + (1 if m.group('branch') else 0)
-        rest = m.group('rest').strip()
+        depth = (len(m.group("indent")) // 3) + (1 if m.group("branch") else 0)
+        rest = m.group("rest").strip()
 
         # First token up to whitespace or ')', optionally starting with '('
-        m2 = re.match(r'^\(?([^\s\)]+)\)?', rest)
+        m2 = re.match(r"^\(?([^\s\)]+)\)?", rest)
         if not m2:
             return None, None
 
-        token = m2.group(1)  # e.g., io.foo:bar:jar:1.2.3:compile
-        parts = token.split(':')
+        token = m2.group(1)  # e.g. com.google.guava:guava:32.0.1-jre:jar:compile
+        parts = token.split(":")
 
-        # Drop trailing Maven scope if present
-        scopes = {'compile', 'runtime', 'test', 'provided', 'system', 'import'}
+        scopes = {"compile", "runtime", "test", "provided", "system", "import"}
         if parts and parts[-1] in scopes:
             parts = parts[:-1]
 
-        # Expect group:artifact:type:(classifier:)version — return without the type
-        if len(parts) >= 4:
-            group, artifact = parts[0], parts[1]
-            version = parts[-1]
-            coord = f"{group}:{artifact}:{version}"
-            return depth, coord
+        if len(parts) < 3:
+            return None, None
+
+        group, artifact = parts[0], parts[1]
+
+        # depgraph text format puts version in position 2:
+        #   group:artifact:version:type   (scope already removed)
+        # We detect that by checking whether the last part is a packaging/type marker.
+        packaging = {"jar", "war", "pom", "bundle", "maven-plugin", "ear", "ejb", "rar", "zip", "test-jar"}
+
+        def looks_like_version(v: str) -> bool:
+            return any(ch.isdigit() for ch in v)
+
+        version: Optional[str] = None
+
+        # depgraph: group:artifact:version:type
+        if len(parts) >= 4 and parts[-1] in packaging and looks_like_version(parts[2]):
+            version = parts[2]
+        # depgraph (rare): group:artifact:version:type:classifier
+        elif len(parts) >= 5 and parts[-2] in packaging and looks_like_version(parts[2]):
+            version = parts[2]
+        else:
+            # Fallback for other Maven-like formats where version is last
+            if looks_like_version(parts[-1]):
+                version = parts[-1]
+            elif looks_like_version(parts[2]):
+                version = parts[2]
+            else:
+                return None, None
 
-        return None, None
+        coord = f"{group}:{artifact}:{version}"
+        return depth, coord
 
 class PythonDependencyTreeBuilder(DependencyTreeBuilder):