feat(scanner): add highlighted packages and contacts extractors (#718)

Author: clemgbld

.changeset/curvy-clouds-drive.md

@@ -0,0 +1,6 @@
+---
+"@nodesecure/contact": minor
+"@nodesecure/scanner": minor
+---
+
+feat(scanner): add highlighted packages and contacts extractors

workspaces/contact/src/ContactExtractor.class.ts

@@ -15,7 +15,7 @@ export type {
 };
 
 export interface ContactExtractorPackageMetadata {
-  author?: Contact;
+  author?: Contact | null;
   maintainers: Contact[];
 }
 
@@ -125,7 +125,7 @@ export class ContactExtractor {
   }
 }
 
-function extractMetadataContacts(
+export function extractMetadataContacts(
   metadata: ContactPackageMetaData
 ): Contact[] {
   return [

workspaces/contact/src/index.ts

@@ -3,3 +3,4 @@ export {
   compareContact
 } from "./utils/index.ts";
 export { NsResolver } from "./NsResolver.class.ts";
+export { UnlitContact } from "./UnlitContact.class.ts";

workspaces/scanner/docs/extractors.md

@@ -39,6 +39,10 @@ Available probes include:
 | Warnings | manifest |
 | Extentions | manifest |
 | NodeDependencies | manifest |
+| HighlightedPackages | manifest |
+| HighlightedContacts | packument |
+
+## ProbeExtractor
 
 All probes follow the same `ProbeExtractor` interface, which acts as an iterator-like contract:
 

workspaces/scanner/src/depWalker.ts

@@ -17,12 +17,11 @@ import {
 } from "@nodesecure/mama";
 import { getNpmRegistryURL } from "@nodesecure/npm-registry-sdk";
 import type Config from "@npmcli/config";
-import semver from "semver";
 
 // Import Internal Dependencies
 import {
-  getDependenciesWarnings,
   addMissingVersionFlags,
+  getDependenciesWarnings,
   getUsedDeps,
   getManifestLinks,
   NPM_TOKEN
@@ -48,7 +47,7 @@ import type {
   Options,
   Payload
 } from "./types.ts";
-import { parseSemverRange } from "./utils/parseSemverRange.ts";
+import { HighlightedPackages } from "./extractors/probes/HighlightedPackagesExtractor.class.ts";
 
 // CONSTANTS
 const kDefaultDependencyVersionFields = {
@@ -185,7 +184,6 @@ export async function depWalker(
   };
 
   const dependencies: Map<string, Dependency> = new Map();
-  const highlightedPackages: Set<string> = new Set();
   const identifiersToHighlight = new Set<string>(options.highlight?.identifiers ?? []);
   const npmTreeWalker = new npm.TreeWalker({
     registry,
@@ -363,6 +361,7 @@ export async function depWalker(
   // We do this because it "seem" impossible to link all dependencies in the first walk.
   // Because we are dealing with package only one time it may happen sometimes.
   const globalWarnings: GlobalWarning[] = [];
+  const highlightedPackagesExtractor = new HighlightedPackages(options.highlight?.packages ?? {});
   for (const [packageName, dependency] of dependencies) {
     const metadataIntegrities = dependency.metadata?.integrity ?? {};
 
@@ -388,22 +387,12 @@ export async function depWalker(
         });
       }
     }
-    const semverRanges = parseSemverRange(options.highlight?.packages ?? {});
     for (const version of Object.entries(dependency.versions)) {
       const [verStr, verDescriptor] = version as [string, DependencyVersion];
-      const packageRange = semverRanges?.[packageName];
-      const org = parseNpmSpec(packageName)?.org;
-      const isScopeHighlighted = org !== null && `@${org}` in semverRanges;
-
-      if (
-        (packageRange && semver.satisfies(verStr, packageRange)) ||
-        isScopeHighlighted
-      ) {
-        highlightedPackages.add(`${packageName}@${verStr}`);
-      }
       verDescriptor.flags.push(
         ...addMissingVersionFlags(new Set(verDescriptor.flags), dependency)
       );
+      highlightedPackagesExtractor.next(verStr, verDescriptor, { name: packageName, dependency });
 
       if (isLocalManifest(verDescriptor, mama, packageName)) {
         const author = mama.author;
@@ -439,9 +428,10 @@ export async function depWalker(
       isRemoteScanning
     );
     payload.warnings = globalWarnings.concat(dependencyConfusionWarnings as GlobalWarning[]).concat(warnings);
+    const { highlightedPackages } = highlightedPackagesExtractor.done();
     payload.highlighted = {
       contacts: illuminated,
-      packages: [...highlightedPackages],
+      packages: highlightedPackages,
       identifiers: extractHighlightedIdentifiers(collectables, identifiersToHighlight)
     };
     payload.dependencies = Object.fromEntries(dependencies);

workspaces/scanner/src/extractors/probes/HighlightedContactsExtractor.class.ts

@@ -0,0 +1,50 @@
+// Import Third-party Dependencies
+import { type EnforcedContact, type IlluminatedContact, UnlitContact, extractMetadataContacts } from "@nodesecure/contact";
+import type { Contact } from "@nodesecure/npm-types";
+
+// Import Internal Dependencies
+import type {
+  PackumentProbeExtractor
+} from "../payload.ts";
+import type { Dependency } from "../../types.ts";
+
+export type HighlightedContactsResult = {
+  illuminated: IlluminatedContact[];
+};
+
+export class HighlightedContacts implements PackumentProbeExtractor<HighlightedContactsResult> {
+  level = "packument" as const;
+
+  #unlitContacts: UnlitContact[];
+
+  constructor(contacts: EnforcedContact[]) {
+    this.#unlitContacts = contacts.map((contact) => new UnlitContact(contact));
+  }
+
+  next(packageName: string, dependency: Dependency) {
+    const extractedContacts = extractMetadataContacts(dependency.metadata);
+    this.addDependencyToUnlitContacts(extractedContacts, packageName);
+  }
+
+  private addDependencyToUnlitContacts(
+    contacts: Contact[],
+    packageName: string
+  ) {
+    for (const unlit of this.#unlitContacts) {
+      const isMaintainer = contacts.some((contact) => unlit.compareTo(contact));
+      if (isMaintainer) {
+        unlit.dependencies.add(packageName);
+      }
+    }
+  }
+
+  done() {
+    const illuminated = this.#unlitContacts.flatMap(
+      (unlit) => (unlit.dependencies.size > 0 ? [unlit.illuminate()] : [])
+    );
+
+    return {
+      illuminated
+    };
+  }
+}

workspaces/scanner/src/extractors/probes/HighlightedPackagesExtractor.class.ts

@@ -0,0 +1,89 @@
+// Import Third-party Dependencies
+import { parseNpmSpec } from "@nodesecure/mama";
+import semver from "semver";
+
+// Import Internal Dependencies
+import type {
+  ManifestProbeExtractor,
+  ProbeExtractorManifestParent
+} from "../payload.ts";
+import type { DependencyVersion, HighlightPackages } from "../../types.ts";
+
+export type HighlightedPackagesResult = {
+  highlightedPackages: string[];
+};
+
+export class HighlightedPackages implements ManifestProbeExtractor<HighlightedPackagesResult> {
+  level = "manifest" as const;
+  #semverRanges: Record<string, string>;
+  #highlightedPackages = new Set<string>();
+
+  constructor(packages: HighlightPackages) {
+    this.#semverRanges = this.#parseSemverRange(packages);
+  }
+
+  #parseSemverRange(packages: HighlightPackages) {
+    const pkgs = Array.isArray(packages) ? this.#parseSpecs(packages) : packages;
+
+    return Object.entries(pkgs).reduce((acc, [name, semverRange]) => {
+      if (Array.isArray(semverRange)) {
+        acc[name] = semverRange.join(" || ");
+      }
+      else {
+        acc[name] = semverRange;
+      }
+
+      return acc;
+    }, {});
+  }
+
+  #parseSpecs(specs: string[]) {
+    return specs.reduce((acc, spec) => {
+      // Handle scope-only entries like "@fastify", matching all packages under that scope
+      if (/^@[^/@]+$/.test(spec)) {
+        acc[spec] = ["*"];
+
+        return acc;
+      }
+
+      const parsedSpec = parseNpmSpec(spec);
+      if (!parsedSpec) {
+        return acc;
+      }
+      const { name, semver } = parsedSpec;
+      const version = semver || "*";
+      if (name in acc) {
+        acc[name].push(version);
+      }
+      else {
+        acc[name] = [version];
+      }
+
+      return acc;
+    }, {});
+  }
+
+  next(
+    version: string,
+    _: DependencyVersion,
+    parent: ProbeExtractorManifestParent
+  ) {
+    const packageRange = this.#semverRanges?.[parent.name];
+    const org = parseNpmSpec(parent.name)?.org;
+    const isScopeHighlighted = org !== null && `@${org}` in this.#semverRanges;
+
+    if (
+      (packageRange && semver.satisfies(version, packageRange)) ||
+      isScopeHighlighted
+    ) {
+      this.#highlightedPackages.add(`${parent.name}@${version}`);
+    }
+  }
+
+  done() {
+    return {
+      highlightedPackages: [...this.#highlightedPackages]
+    };
+  }
+}
+

workspaces/scanner/src/extractors/probes/index.ts

@@ -6,3 +6,5 @@ export * from "./VulnerabilitiesExtractor.class.ts";
 export * from "./FlagsExtractor.class.ts";
 export * from "./ExtensionsExtractor.class.ts";
 export * from "./NodeDependenciesExtractor.class.ts";
+export * from "./HighlightedPackagesExtractor.class.ts";
+export * from "./HighlightedContactsExtractor.class.ts";