Add support for download URL as --input-list in batch-create #1524

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

docs/command-line-interface.rst

@@ -174,6 +174,10 @@ Required arguments (one of):
   | project-2      | pkg:deb/debian/curl@7.50.3      |
   +----------------+---------------------------------+
 
+.. tip::
+    In place of a local path, a download URL to the CSV file is supported for the
+    ``--input-list`` argument.
+
 Optional arguments:
 
 - ``--project-name-suffix`` Optional custom suffix to append to project names.
@@ -194,14 +198,15 @@ Optional arguments:
 Example: Processing Multiple Docker Images
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Assume multiple Docker images are available in a directory named ``local-data/`` on
+Suppose you have multiple Docker images stored in a directory named ``local-data/`` on
 the host machine.
-To process these images with the ``analyze_docker_image`` pipeline using asynchronous
-execution::
+To process these images using the ``analyze_docker_image`` pipeline with asynchronous
+execution, you can use this command::
 
     $ docker compose run --rm \
-        --volume local-data/:/input-data:ro \
-        web scanpipe batch-create input-data/ \
+        --volume local-data/:/input-data/:ro \
+        web scanpipe batch-create
+            --input-directory /input-data/ \
             --pipeline analyze_docker_image \
             --label "Docker" \
             --execute --async
@@ -224,6 +229,19 @@ Each Docker image in the ``local-data/`` directory will result in the creation o
 project with the specified pipeline (``analyze_docker_image``) executed by worker
 services.
 
+Example: Processing Multiple Develop to Deploy Mapping
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To process an input list CSV file with the ``map_deploy_to_develop`` pipeline using
+asynchronous execution::
+
+    $ docker compose run --rm \
+        web scanpipe batch-create \
+            --input-list https://url/input_list.csv \
+            --pipeline map_deploy_to_develop \
+            --label "d2d_mapping" \
+            --execute --async
+
 `$ scanpipe list-pipeline [--verbosity {0,1,2,3}]`
 --------------------------------------------------
 

scanpipe/management/commands/batch-create.py

@@ -27,8 +27,11 @@
 from django.core.management import CommandError
 from django.core.management.base import BaseCommand
 
+import requests
+
 from scanpipe.management.commands import CreateProjectCommandMixin
 from scanpipe.management.commands import PipelineCommandMixin
+from scanpipe.pipes import fetch
 
 
 class Command(CreateProjectCommandMixin, PipelineCommandMixin, BaseCommand):
@@ -54,7 +57,8 @@ def add_arguments(self, parser):
                 "Path to a CSV file with project names and input URLs. "
                 "The first column must contain project names, and the second column "
                 "should list comma-separated input URLs (e.g., Download URL, PURL, or "
-                "Docker reference)."
+                "Docker reference). "
+                "In place of a local path, a download URL to the CSV file is supported."
             ),
         )
         parser.add_argument(
@@ -110,7 +114,16 @@ def handle_input_directory(self, **options):
                 self.created_project_count += 1
 
     def handle_input_list(self, **options):
-        input_file = Path(options["input_list"])
+        input_file = options["input_list"]
+
+        if input_file.startswith("http"):
+            try:
+                download = fetch.fetch_http(input_file)
+            except requests.exceptions.RequestException as e:
+                raise CommandError(e)
+            input_file = download.path
+
+        input_file = Path(input_file)
         if not input_file.exists():
             raise CommandError(f"The {input_file} file does not exist.")