Wire batch flags into scrape/search/screenshot/codegen

Attach --input-file, --input-column and --concurrency to every
scrape-style command and dispatch into the batch pipeline when
--input-file is present.

- createTargetAction detects batch mode, rejects --input-file combined
  with a positional input, and otherwise runs each input through the
  same body builder + SDK call with bounded concurrency.
- Primary positional args are now optional so the input can come from
  the file instead; a genuinely missing input raises a CliUsageError.
- Binary targets (screenshot) write one file per item and require -o
  <dir>; non-binary targets stream ndjson or write per-item JSON files.

Integration tests cover flag exposure and the usage-error paths; the
orchestrator is unit-tested end to end without network calls.

Author: paulius-krutkis-dcd

src/batch/commands/attach-batch-options.ts

@@ -0,0 +1,19 @@
+import type { Command } from "commander";
+import { parseConcurrency } from "../services/resolve-concurrency.js";
+
+export function attachBatchOptions(command: Command): Command {
+  return command
+    .option(
+      "--input-file <path>",
+      "Run each line/row of a .txt or .csv file as a batch item"
+    )
+    .option(
+      "--input-column <name>",
+      "Column to read inputs from when --input-file is a CSV"
+    )
+    .option(
+      "--concurrency <n>",
+      "Max requests to run in parallel in batch mode (default: 4)",
+      parseConcurrency
+    );
+}

src/batch/services/binary-directory-sink.ts

@@ -0,0 +1,39 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { BatchResult } from "../types/batch-result.js";
+import type { BatchSink } from "../types/batch-sink.js";
+import { batchItemFilename } from "./batch-item-filename.js";
+import { toBatchRecord } from "./batch-record.js";
+import { uniqueName } from "./unique-name.js";
+
+function isBytes(value: unknown): value is Uint8Array {
+  return value instanceof Uint8Array;
+}
+
+export function createBinaryDirectorySink(
+  dir: string,
+  extension = "png"
+): BatchSink {
+  mkdirSync(dir, { recursive: true });
+  const used = new Set<string>();
+
+  return {
+    write(result: BatchResult): void {
+      const name = uniqueName(
+        batchItemFilename(result.input, result.index),
+        used
+      );
+
+      if (result.ok && isBytes(result.data)) {
+        writeFileSync(join(dir, `${name}.${extension}`), result.data);
+        return;
+      }
+
+      writeFileSync(
+        join(dir, `${name}.error.json`),
+        JSON.stringify(toBatchRecord(result), null, 2),
+        "utf8"
+      );
+    },
+  };
+}

src/batch/services/resolve-concurrency.ts

@@ -0,0 +1,14 @@
+import { ValidationError } from "@decodo/sdk-ts";
+import { DEFAULT_CONCURRENCY } from "../constants.js";
+
+export function parseConcurrency(value: string): number {
+  const parsed = Number.parseInt(value, 10);
+  if (Number.isNaN(parsed) || parsed < 1) {
+    throw new ValidationError("--concurrency must be a positive integer.");
+  }
+  return parsed;
+}
+
+export function resolveConcurrency(value: number | undefined): number {
+  return value ?? DEFAULT_CONCURRENCY;
+}

src/batch/services/run-batch-command.ts

@@ -0,0 +1,64 @@
+import { ValidationError } from "@decodo/sdk-ts";
+import { CliUsageError } from "../../platform/services/handle-cli-error.js";
+import type { BatchSummary } from "../types/batch-result.js";
+import type { BatchSink } from "../types/batch-sink.js";
+import { createBinaryDirectorySink } from "./binary-directory-sink.js";
+import { createDirectorySink } from "./directory-sink.js";
+import { createNdjsonStdoutSink } from "./ndjson-stdout-sink.js";
+import { readInputFile } from "./read-input-file.js";
+import { runBatch } from "./run-batch.js";
+
+export interface RunBatchCommandOptions {
+  binary?: boolean;
+  concurrency: number;
+  inputColumn?: string;
+  inputFile: string;
+  output?: string;
+  pretty?: boolean;
+  scrapeItem: (input: string) => Promise<unknown>;
+}
+
+function selectSink(options: RunBatchCommandOptions): BatchSink {
+  if (options.binary) {
+    if (!options.output) {
+      throw new CliUsageError(
+        "Batch mode for binary output requires -o <dir> to write files."
+      );
+    }
+    return createBinaryDirectorySink(options.output);
+  }
+
+  if (options.output) {
+    return createDirectorySink(options.output, { pretty: options.pretty });
+  }
+
+  return createNdjsonStdoutSink();
+}
+
+export async function runBatchCommand(
+  options: RunBatchCommandOptions
+): Promise<BatchSummary> {
+  const sink = selectSink(options);
+  const items = readInputFile(options.inputFile, {
+    inputColumn: options.inputColumn,
+  });
+
+  const summary = await runBatch({
+    items,
+    concurrency: options.concurrency,
+    worker: (item) => options.scrapeItem(item.input),
+    onResult: (result) => sink.write(result),
+  });
+
+  await sink.close?.();
+
+  if (summary.total === 0) {
+    throw new ValidationError("Input file produced no inputs.");
+  }
+
+  console.error(
+    `Batch complete: ${summary.succeeded} succeeded, ${summary.failed} failed (${summary.total} total).`
+  );
+
+  return summary;
+}

src/batch/types/batch-flags.ts

@@ -0,0 +1,5 @@
+export interface BatchFlags {
+  concurrency?: number;
+  inputColumn?: string;
+  inputFile?: string;
+}

src/scrape/commands/scrape.ts

@@ -1,8 +1,10 @@
 import { type DecodoSchema, Target, ValidationError } from "@decodo/sdk-ts";
 import { Command } from "commander";
+import { attachBatchOptions } from "../../batch/commands/attach-batch-options.js";
 import { attachScrapeOutputOptions } from "../../output/commands/attach-output-options.js";
 import { applyRequestDefaults } from "../../output/services/apply-request-defaults.js";
 import type { OutputOptions } from "../../output/types/output-options.js";
+import { CliUsageError } from "../../platform/services/handle-cli-error.js";
 import { resolveTarget } from "../services/resolve-target.js";
 import { createTargetAction } from "../services/run-target-scrape.js";
 import type { ScrapeOptions } from "../types/scrape-command.js";
@@ -32,17 +34,18 @@ export function createScrapeCommand(schema: DecodoSchema): Command {
     .description(
       "Scrape a URL with the universal target (markdown by default). Use decodo universal for --markdown, --parse, and other API flags."
     )
-    .argument("<url>", "URL to scrape")
+    .argument("[url]", "URL to scrape (omit when using --input-file)")
     .option("--country <code>", "Geo / country code (maps to geo)")
     .option("--headers <json>", "Request headers as a JSON object string")
     .option("--target <name>", "Scrape target override (default: universal)");
 
   attachScrapeOutputOptions(command);
+  attachBatchOptions(command);
 
   return command.action(
     createTargetAction(Target.Universal, schema, (url, options) => {
       if (url === undefined) {
-        throw new Error("Missing required URL.");
+        throw new CliUsageError("Missing required URL.");
       }
 
       const opts = options as ScrapeOptions & OutputOptions;

src/scrape/commands/screenshot.ts

@@ -1,6 +1,8 @@
 import { type DecodoSchema, Target } from "@decodo/sdk-ts";
 import { Command } from "commander";
+import { attachBatchOptions } from "../../batch/commands/attach-batch-options.js";
 import { attachScrapeOutputOptions } from "../../output/commands/attach-output-options.js";
+import { CliUsageError } from "../../platform/services/handle-cli-error.js";
 import { resolveTarget } from "../services/resolve-target.js";
 import { createTargetAction } from "../services/run-target-scrape.js";
 import type { ScreenshotOptions } from "../types/screenshot-command.js";
@@ -10,21 +12,22 @@ export function createScreenshotCommand(schema: DecodoSchema): Command {
     .description(
       "Capture a PNG screenshot (universal, headless). Use decodo universal --headless png for full options."
     )
-    .argument("<url>", "URL to screenshot")
+    .argument("[url]", "URL to screenshot (omit when using --input-file)")
     .option("--country <code>", "Geo / country code (maps to geo)")
     .option("--target <name>", "Scrape target override (default: universal)");
 
   attachScrapeOutputOptions(command, {
     outputHelp: "Write PNG to file or directory (default name: <host>.png)",
   });
+  attachBatchOptions(command);
 
   return command.action(
     createTargetAction(
       Target.Universal,
       schema,
       (url, options) => {
         if (url === undefined) {
-          throw new Error("Missing required URL.");
+          throw new CliUsageError("Missing required URL.");
         }
 
         const opts = options as ScreenshotOptions;

src/scrape/commands/search.ts

@@ -1,7 +1,9 @@
 import { type DecodoSchema, Target, ValidationError } from "@decodo/sdk-ts";
 import { Command, Option } from "commander";
+import { attachBatchOptions } from "../../batch/commands/attach-batch-options.js";
 import { attachScrapeOutputOptions } from "../../output/commands/attach-output-options.js";
 import { applyRequestDefaults } from "../../output/services/apply-request-defaults.js";
+import { CliUsageError } from "../../platform/services/handle-cli-error.js";
 import { resolveTarget } from "../services/resolve-target.js";
 import { createTargetAction } from "../services/run-target-scrape.js";
 import type { SearchOptions } from "../types/search-command.js";
@@ -52,7 +54,7 @@ export function createSearchCommand(schema: DecodoSchema): Command {
     .description(
       "Search the web (default: Google). Use decodo google-search or decodo bing-search for full options."
     )
-    .argument("<query>", "Search query")
+    .argument("[query]", "Search query (omit when using --input-file)")
     .addOption(
       new Option("--engine <engine>", "Search engine")
         .choices(["google", "bing"])
@@ -63,11 +65,12 @@ export function createSearchCommand(schema: DecodoSchema): Command {
     .option("--target <name>", "Scrape target override");
 
   attachScrapeOutputOptions(command);
+  attachBatchOptions(command);
 
   return command.action(
     createTargetAction(Target.GoogleSearch, schema, (query, options) => {
       if (query === undefined) {
-        throw new Error("Missing required query.");
+        throw new CliUsageError("Missing required query.");
       }
 
       const opts = options as SearchOptions;

src/scrape/services/command-builder.ts

@@ -1,8 +1,10 @@
 import type { DecodoSchema } from "@decodo/sdk-ts";
 import { type Command, Option } from "commander";
 import type { JSONSchema4 } from "json-schema";
+import { attachBatchOptions } from "../../batch/commands/attach-batch-options.js";
 import { attachScrapeOutputOptions } from "../../output/commands/attach-output-options.js";
 import { applyRequestDefaults } from "../../output/services/apply-request-defaults.js";
+import { CliUsageError } from "../../platform/services/handle-cli-error.js";
 import type { TargetCommandConfig } from "../types/target-command.js";
 import { snakeToCamel, snakeToKebab } from "./naming.js";
 import { getPrimaryInputField } from "./primary-input.js";
@@ -81,7 +83,7 @@ export function configureTargetCommand(
       | undefined;
     const inputHelp =
       primarySchema?.description ?? `Primary ${primaryField} input`;
-    command.argument("<input>", inputHelp);
+    command.argument("[input]", inputHelp);
   }
 
   const optionFields = Object.keys(parameterSchema?.properties ?? {}).filter(
@@ -94,6 +96,7 @@ export function configureTargetCommand(
   }
 
   attachScrapeOutputOptions(command);
+  attachBatchOptions(command);
 
   return { target, primaryField, optionFields };
 }
@@ -109,7 +112,9 @@ export function buildScrapeBody(
 
   if (config.primaryField) {
     if (input === undefined) {
-      throw new Error(`Missing required input for ${config.primaryField}.`);
+      throw new CliUsageError(
+        `Missing required input for ${config.primaryField}.`
+      );
     }
     body[config.primaryField] = input;
   }

src/scrape/services/run-target-scrape.ts

@@ -2,18 +2,26 @@ import type { DecodoSchema, ScrapeRequest } from "@decodo/sdk-ts";
 import type { Command } from "commander";
 import { AuthRequiredError } from "../../auth/errors/auth-required-error.js";
 import { resolveAuthToken } from "../../auth/services/resolve-token.js";
+import { resolveConcurrency } from "../../batch/services/resolve-concurrency.js";
+import { runBatchCommand } from "../../batch/services/run-batch-command.js";
+import type { BatchFlags } from "../../batch/types/batch-flags.js";
 import { getRootOpts } from "../../cli/services/global-opts.js";
 import { verboseLog } from "../../cli/services/verbose-log.js";
+import { extractPayload } from "../../output/services/extract-payload.js";
 import { writeScrapeResponse } from "../../output/services/write-scrape-response.js";
 import type { OutputOptions } from "../../output/types/output-options.js";
-import { handleCliError } from "../../platform/services/handle-cli-error.js";
+import {
+  CliUsageError,
+  handleCliError,
+} from "../../platform/services/handle-cli-error.js";
 import type {
   ExecuteScrapeOptions,
   OutputContextBuilder,
   ScrapeBodyBuilder,
 } from "../types/run-target-scrape.js";
 import { createDecodoClient } from "./client.js";
 import { buildScrapeBody, getTargetCommandConfig } from "./command-builder.js";
+import { extractPngFromResponse } from "./extract-png.js";
 import { formatScrapeRequestLog } from "./format-scrape-request-log.js";
 
 async function executeScrape({
@@ -39,6 +47,47 @@ async function executeScrape({
   });
 }
 
+interface ExecuteBatchOptions {
+  binary: boolean;
+  options: Record<string, unknown>;
+  resolveBody: ScrapeBodyBuilder;
+  schema: DecodoSchema;
+  token: string;
+  verbose: boolean;
+}
+
+async function executeBatch({
+  token,
+  schema,
+  options,
+  resolveBody,
+  binary,
+  verbose,
+}: ExecuteBatchOptions): Promise<void> {
+  const client = createDecodoClient(token, schema);
+  const batch = options as BatchFlags & OutputOptions;
+  const full = batch.full === true;
+
+  await runBatchCommand({
+    inputFile: batch.inputFile as string,
+    inputColumn: batch.inputColumn,
+    concurrency: resolveConcurrency(batch.concurrency),
+    output: batch.output,
+    pretty: batch.pretty,
+    binary,
+    scrapeItem: async (itemInput) => {
+      const body = resolveBody(itemInput, options);
+      verboseLog(verbose, formatScrapeRequestLog(body));
+      const response = await client.webScrapingApi.scrape(
+        body as unknown as ScrapeRequest
+      );
+      return binary
+        ? extractPngFromResponse(response)
+        : extractPayload(response, full);
+    },
+  });
+}
+
 export function createTargetAction(
   target: string,
   schema: DecodoSchema,
@@ -60,12 +109,35 @@ export function createTargetAction(
     const verbose = rootOpts.verbose === true;
 
     try {
+      const batchMode = (options as BatchFlags).inputFile !== undefined;
+      if (batchMode && input !== undefined) {
+        throw new CliUsageError(
+          "Cannot combine --input-file with a positional input."
+        );
+      }
+      if (!batchMode && config.primaryField && input === undefined) {
+        throw new CliUsageError(`Missing required ${config.primaryField}.`);
+      }
+
       const auth = await resolveAuthToken({ token: rootOpts.token });
       verboseLog(verbose, `auth source=${auth.source}`);
       if (!auth.token) {
         throw new AuthRequiredError();
       }
 
+      if (batchMode) {
+        const outputContext = getOutputContext?.(undefined, options);
+        await executeBatch({
+          token: auth.token,
+          schema,
+          options,
+          resolveBody,
+          binary: outputContext?.binary?.kind === "png",
+          verbose,
+        });
+        return;
+      }
+
       const body = resolveBody(input, options);
       verboseLog(verbose, formatScrapeRequestLog(body));
       const outputContext = getOutputContext?.(input, options);