Refactor alpine mining pipeline for git deployment

Resolves: #774

Signed-off-by: Keshav Priyadarshi <git@keshav.space>

Author: keshav-space

minecode_pipelines/pipelines/mine_alpine.py

@@ -20,36 +20,29 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from scanpipe.pipelines import Pipeline
-from scanpipe.pipes import federatedcode
+from minecode_pipelines.pipelines import MineCodeBasePipeline
 
-from minecode_pipelines import pipes
 from minecode_pipelines.pipes import alpine
 
 
-class MineAlpine(Pipeline):
-    """
-    Mine all packageURLs from an alpine index and publish them to
-    a FederatedCode repo.
-    """
+class MineAlpine(MineCodeBasePipeline):
+    """Mine PackageURLs from alpine index and publish them to FederatedCode."""
 
     @classmethod
     def steps(cls):
         return (
             cls.check_federatedcode_eligibility,
-            cls.collect_packages_from_alpine,
-            cls.delete_cloned_repos,
+            cls.create_federatedcode_working_dir,
+            cls.fetch_federation_config,
+            cls.mine_and_publish_alpine_packageurls,
+            cls.delete_working_dir,
         )
 
-    def check_federatedcode_eligibility(self):
-        """
-        Check if the project fulfills the following criteria for
-        pushing the project result to FederatedCode.
-        """
-        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
-
-    def collect_packages_from_alpine(self):
-        self.repos = alpine.collect_packages_from_alpine(logger=self.log)
-
-    def delete_cloned_repos(self):
-        pipes.delete_cloned_repos(repos=self.repos, logger=self.log)
+    def mine_and_publish_alpine_packageurls(self):
+        alpine.mine_and_publish_alpine_packageurls(
+            data_cluster=self.data_cluster,
+            checked_out_repos=self.checked_out_repos,
+            working_path=self.working_path,
+            commit_msg_func=self.commit_message,
+            logger=self.log,
+        )

minecode_pipelines/pipes/alpine.py

@@ -23,21 +23,17 @@
 import base64
 from shutil import rmtree
 
-from aboutcode import hashid
+from aboutcode.pipeline import LoopProgress
 from packagedcode.models import PackageData
 from packagedcode.models import Party
 from packageurl import PackageURL
-from scanpipe.pipes import federatedcode
 from scanpipe.pipes.fetch import fetch_http
 from scanpipe.pipes.scancode import extract_archives
 
-from minecode_pipelines import pipes
-from minecode_pipelines import VERSION
+from minecode_pipelines.pipelines import _mine_and_publish_packageurls
 
 ALPINE_CHECKPOINT_PATH = "alpine/checkpoints.json"
 
-# We are testing and storing mined packageURLs in one single repo per ecosystem for now
-MINECODE_DATA_ALPINE_REPO = "https://github.com/aboutcode-data/minecode-data-alpine-test"
 
 # Number of packages
 PACKAGE_BATCH_SIZE = 1000
@@ -530,86 +526,52 @@ def _fetch_index(self, uri):
     def get_packages(self, logger=None):
         """Yield Package objects from alpine index"""
         for apkindex_url in ALPINE_LINUX_APKINDEX_URLS:
-            _, subpath = apkindex_url.split("https://dl-cdn.alpinelinux.org/alpine/")
-            distro, repo, _, _ = subpath.split("/")
-            index = self._fetch_index(uri=apkindex_url)
-            extract_archives(location=index.path)
-            index_location = f"{index.path}-extract/APKINDEX"
-            with open(index_location, encoding="utf-8") as f:
-                for pkg in parse_apkindex(f.read()):
-                    pd = build_package(pkg, distro=distro, repo=repo)
-                    current_purl = PackageURL(
-                        type=pd.type,
-                        namespace=pd.namespace,
-                        name=pd.name,
-                    )
-                    yield current_purl, pd
-
-
-def commit_message(commit_batch, total_commit_batch="many"):
-    from django.conf import settings
-
-    author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME
-    author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL
-    tool_name = "pkg:github/aboutcode-org/scancode.io"
-
-    return f"""\
-        Collect PackageURLs from Alpine ({commit_batch}/{total_commit_batch})
-
-        Tool: {tool_name}@v{VERSION}
-        Reference: https://{settings.ALLOWED_HOSTS[0]}
-
-        Signed-off-by: {author_name} <{author_email}>
-        """
-
-
-def collect_packages_from_alpine(files_per_commit=PACKAGE_BATCH_SIZE, logger=None):
-    # Clone data and config repo
-    data_repo = federatedcode.clone_repository(
-        repo_url=MINECODE_DATA_ALPINE_REPO,
+            self.get_package_from_index(apkindex_url)
+
+    def get_package_from_index(self, apkindex_url, logger=None):
+        _, subpath = apkindex_url.split("https://dl-cdn.alpinelinux.org/alpine/")
+        distro, repo, _, _ = subpath.split("/")
+        index = self._fetch_index(uri=apkindex_url)
+        extract_archives(location=index.path)
+        index_location = f"{index.path}-extract/APKINDEX"
+        with open(index_location, encoding="utf-8") as f:
+            for pkg in parse_apkindex(f.read()):
+                pd = build_package(pkg, distro=distro, repo=repo)
+                current_purl = PackageURL(
+                    type=pd.type,
+                    namespace=pd.namespace,
+                    name=pd.name,
+                )
+                yield current_purl, [pd.purl]
+
+
+def mine_and_publish_alpine_packageurls(
+    data_cluster,
+    checked_out_repos,
+    working_path,
+    commit_msg_func,
+    logger,
+):
+    """Yield PackageURLs from Alpine index."""
+
+    index_count = len(ALPINE_LINUX_APKINDEX_URLS)
+    progress = LoopProgress(
+        total_iterations=index_count,
         logger=logger,
+        progress_step=1,
     )
-    config_repo = federatedcode.clone_repository(
-        repo_url=pipes.MINECODE_PIPELINES_CONFIG_REPO,
-        logger=logger,
-    )
-    if logger:
-        logger(f"{MINECODE_DATA_ALPINE_REPO} repo cloned at: {data_repo.working_dir}")
-        logger(f"{pipes.MINECODE_PIPELINES_CONFIG_REPO} repo cloned at: {config_repo.working_dir}")
 
-    # download and iterate through alpine indices
+    logger(f"Mine PackageURL from {index_count:,d} alpine index.")
     alpine_collector = AlpineCollector()
-    files_to_commit = []
-    commit_batch = 1
-    for current_purl, package in alpine_collector.get_packages():
-        # write packageURL to file
-        package_base_dir = hashid.get_package_base_dir(purl=current_purl)
-        purl_file = pipes.write_packageurls_to_file(
-            repo=data_repo,
-            base_dir=package_base_dir,
-            packageurls=[package.purl],
-            append=True,
-        )
-        if purl_file not in files_to_commit:
-            files_to_commit.append(purl_file)
-
-        if len(files_to_commit) == files_per_commit:
-            federatedcode.commit_and_push_changes(
-                commit_message=commit_message(commit_batch),
-                repo=data_repo,
-                files_to_commit=files_to_commit,
-                logger=logger,
-            )
-            files_to_commit.clear()
-            commit_batch += 1
-
-    if files_to_commit:
-        federatedcode.commit_and_push_changes(
-            commit_message=commit_message(commit_batch),
-            repo=data_repo,
-            files_to_commit=files_to_commit,
+    for index in progress.iter(ALPINE_LINUX_APKINDEX_URLS):
+        logger(f"Mine PackageURL from {index} index.")
+        _mine_and_publish_packageurls(
+            packageurls=alpine_collector.get_package_from_index(index),
+            total_package_count=None,
+            data_cluster=data_cluster,
+            checked_out_repos=checked_out_repos,
+            working_path=working_path,
+            append_purls=True,
+            commit_msg_func=commit_msg_func,
             logger=logger,
         )
-
-    repos_to_clean = [data_repo, config_repo]
-    return repos_to_clean

pyproject-minecode_pipelines.toml

@@ -4,7 +4,7 @@ build-backend = "flot.buildapi"
 
 [project]
 name = "minecode_pipelines"
-version = "0.0.1b27"
+version = "0.0.1b30"
 description = "A library for mining packageURLs and package metadata from ecosystem repositories."
 readme = "minecode_pipelines/README.rst"
 license = { text = "Apache-2.0" }