make shell_command targets be runnable via the run goal (#22981)

Teach the shell backend how to run `shell_command` targets via the `run`
goal and make them valid "runnable" targets for other Pants targets
which can reference runnables including `code_quality_tool`. Running a
`shell_command` target can take place in the Pants execution sandbox.

This PR intentionally does not remove nor deprecate `run_shell_command`
and its more limited execution model of only supporting interactive
execution. `run_shell_command` does not have all the fields of
`shell_command` and adding such support seemed a less desirable UX
change than just making `shell_command` be runnable. At the very, this
PR is an incremental improvement over the existing situation and defers
needing to make decision on `run_shell_command` (and the other
`FOO_shell_command` target types) at this time.

Author: tdyas

docs/notes/2.31.x.md

@@ -73,6 +73,8 @@ Upgraded the default version of `protoc` to v30.2. Python projects should upgrad
 
 #### Shell
 
+`shell_command` targets are now runnable via the `run` goal and can be used as a "runnable" target by other targets including the `code_quality_tool` target type. The `run_shell_command` target type remains available at the moment. The difference between running a `shell_command` versus a `run_shell_command` target is that `shell_command` targets can be run within the Pants execution sandbox (and outputs captured) while the more limited execution model of the `run_shell_command` target type only supports interactive execution.
+
 #### Javascript
 
 #### TypeScript

src/python/pants/backend/shell/target_types.py

@@ -269,7 +269,7 @@ class ShellSourcesGeneratorTarget(TargetFilesGenerator):
 # -----------------------------------------------------------------------------------------------
 
 
-class ShellCommandCommandField(StringField):
+class ShellCommandCommandFieldBase(StringField):
     alias = "command"
     required = True
     help = help_text(
@@ -283,6 +283,10 @@ class ShellCommandCommandField(StringField):
     )
 
 
+class ShellCommandCommandField(ShellCommandCommandFieldBase):
+    pass
+
+
 class ShellCommandOutputFilesField(AdhocToolOutputFilesField):
     pass
 
@@ -438,7 +442,14 @@ class ShellCommandTarget(Target):
     )
     help = help_text(
         """
-        Execute any external tool for its side effects.
+        Execute any external tool for its side effects, or run it interactively.
+
+        This target provides hermetic execution with explicit tool dependencies via the
+        `tools` field. It can be used for:
+
+        - Code generation (produces output files consumed by other targets)
+        - Running scripts interactively with explicit dependencies (via `pants run`)
+        - Build-time hermetic execution (via `pants experimental_run_in_sandbox`)
 
         Example BUILD file:
 
@@ -452,27 +463,39 @@ class ShellCommandTarget(Target):
 
             shell_sources(name="scripts")
 
-        Remember to add this target to the dependencies of each consumer, such as your
-        `python_tests` or `docker_image`. When relevant, Pants will run your `command` and
-        insert the `outputs` into that consumer's context.
+        When used as a dependency of other targets (e.g., `python_tests` or `docker_image`),
+        Pants will run your `command` and insert the `outputs` into that consumer's context.
+
+        When used with `pants run :target`, the command runs interactively in the workspace
+        with all dependencies and tools available.
 
         The command may be retried and/or cancelled, so ensure that it is idempotent.
+
+        For simpler, workspace-oriented scripts that use system PATH tools, consider
+        `run_shell_command` instead.
         """
     )
 
 
+class RunShellCommandCommandField(ShellCommandCommandFieldBase):
+    pass
+
+
 class ShellCommandRunTarget(Target):
     alias = "run_shell_command"
     core_fields = (
         *COMMON_TARGET_FIELDS,
         RunShellCommandExecutionDependenciesField,
         RunShellCommandRunnableDependenciesField,
-        ShellCommandCommandField,
+        RunShellCommandCommandField,
         RunShellCommandWorkdirField,
     )
     help = help_text(
         """
-        Run a script in the workspace, with all dependencies packaged/copied into a chroot.
+        Run a script in the workspace with dependencies packaged into a chroot.
+
+        This target is designed for quick, workspace-oriented interactive scripts that use
+        tools from the system PATH.
 
         Example BUILD file:
 
@@ -484,10 +507,13 @@ class ShellCommandRunTarget(Target):
         The `command` may use either `{chroot}` on the command line, or the `$CHROOT`
         environment variable to get the root directory for where any dependencies are located.
 
-        In contrast to the `shell_command`, in addition to `workdir` you only have
-        the `command` and `execution_dependencies` fields as the `tools` you are going to use are
-        already on the PATH which is inherited from the Pants environment. Also, the `outputs` does
-        not apply, as any output files produced will end up directly in your project tree.
+        In contrast to `shell_command`, this target:
+        - Uses tools from the system PATH (not explicit `tools` field)
+        - Does not support `output_files` (outputs go directly to workspace)
+        - Is simpler to use for quick workspace scripts
+
+        For more hermetic execution with explicit tool dependencies, consider using
+        `shell_command` instead, which provides better reproducibility and caching.
         """
     )
 

src/python/pants/backend/shell/util_rules/shell_command.py

@@ -10,9 +10,11 @@
 
 from pants.backend.shell.subsystems.shell_setup import ShellSetup
 from pants.backend.shell.target_types import (
+    RunShellCommandCommandField,
     RunShellCommandWorkdirField,
     ShellCommandCacheScopeField,
     ShellCommandCommandField,
+    ShellCommandCommandFieldBase,
     ShellCommandExecutionDependenciesField,
     ShellCommandExtraEnvVarsField,
     ShellCommandLogOutputField,
@@ -98,7 +100,7 @@ async def prepare_process_request_from_target(
     working_directory = shell_command[ShellCommandWorkdirField].value
     assert working_directory is not None, "working_directory should always be a string"
 
-    command = shell_command[ShellCommandCommandField].value
+    command = shell_command[ShellCommandCommandFieldBase].value
     if not command:
         raise ValueError(f"Missing `command` line in `{description}.")
 
@@ -214,12 +216,26 @@ async def prepare_process_request_from_target(
 
 class RunShellCommand(RunFieldSet):
     required_fields = (
-        ShellCommandCommandField,
+        RunShellCommandCommandField,
         RunShellCommandWorkdirField,
     )
     run_in_sandbox_behavior = RunInSandboxBehavior.NOT_SUPPORTED
 
 
+@dataclass(frozen=True)
+class RunShellCommandBuild(RunFieldSet):
+    """Run a shell_command target interactively with explicit tool dependencies."""
+
+    required_fields = (ShellCommandCommandField,)
+    run_in_sandbox_behavior = RunInSandboxBehavior.RUN_REQUEST_HERMETIC
+
+    command: ShellCommandCommandField
+    execution_dependencies: ShellCommandExecutionDependenciesField
+    runnable_dependencies: ShellCommandRunnableDependenciesField
+    tools: ShellCommandToolsField
+    workdir: ShellCommandWorkdirField
+
+
 @rule(desc="Running shell command", level=LogLevel.DEBUG)
 async def shell_command_in_sandbox(
     request: GenerateFilesFromShellCommandRequest,
@@ -255,7 +271,7 @@ async def _interactive_shell_command(
     if working_directory is None:
         raise ValueError("Working directory must be not be `None` for interactive processes.")
 
-    command = shell_command[ShellCommandCommandField].value
+    command = shell_command[RunShellCommandCommandField].value
     if not command:
         raise ValueError(f"Missing `command` line in `{description}.")
 
@@ -303,12 +319,110 @@ async def run_shell_command_request(bash: BashBinary, shell_command: RunShellCom
     )
 
 
+@rule(desc="Running shell_command target", level=LogLevel.DEBUG)
+async def run_shell_command_build_request(
+    field_set: RunShellCommandBuild,
+    bash: BashBinary,
+    shell_setup: ShellSetup.EnvironmentAware,
+) -> RunRequest:
+    """Execute a shell_command target interactively with explicit tool dependencies."""
+
+    command = field_set.command.value
+    if not command:
+        raise ValueError(
+            f"Missing `command` field in `shell_command` target at `{field_set.address}`."
+        )
+
+    # Resolve execution environment with all dependencies
+    execution_environment = await resolve_execution_environment(
+        ResolveExecutionDependenciesRequest(
+            field_set.address,
+            field_set.execution_dependencies.value,
+            field_set.runnable_dependencies.value,
+        ),
+        **implicitly(),
+    )
+
+    # Resolve tools into binary shims
+    tools = field_set.tools.value or ()
+    tools = tuple(tool for tool in tools if tool not in BASH_BUILTIN_COMMANDS)
+    resolved_tools = await create_binary_shims(
+        BinaryShimsRequest.for_binaries(
+            *tools,
+            rationale=f"execute `shell_command` at `{field_set.address}`",
+            search_path=shell_setup.executable_search_path,
+        ),
+        bash,
+    )
+
+    # Prepare extra sandbox contents with tools and runnable dependencies
+    runnable_dependencies = execution_environment.runnable_dependencies
+    extra_sandbox_contents: list[ExtraSandboxContents] = []
+
+    # Add tools to the environment
+    extra_sandbox_contents.append(
+        ExtraSandboxContents(
+            digest=EMPTY_DIGEST,
+            paths=(resolved_tools.path_component,),
+            immutable_input_digests=FrozenDict(resolved_tools.immutable_input_digests or {}),
+            append_only_caches=FrozenDict(),
+            extra_env=FrozenDict(),
+        )
+    )
+
+    # Add runnable dependencies
+    if runnable_dependencies:
+        extra_sandbox_contents.append(
+            ExtraSandboxContents(
+                digest=EMPTY_DIGEST,
+                paths=(f"{{chroot}}/{runnable_dependencies.path_component}",),
+                immutable_input_digests=runnable_dependencies.immutable_input_digests,
+                append_only_caches=runnable_dependencies.append_only_caches,
+                extra_env=runnable_dependencies.extra_env,
+            )
+        )
+
+    merged_extras = await merge_extra_sandbox_contents(
+        MergeExtraSandboxContents(tuple(extra_sandbox_contents))
+    )
+
+    # Prepare environment variables
+    env_vars = await prepare_env_vars(
+        merged_extras.extra_env,
+        (),  # No extra env vars field for run
+        extra_paths=merged_extras.paths,
+        description_of_origin=f"`shell_command` target at `{field_set.address}`",
+    )
+
+    # Parse working directory
+    working_directory = field_set.workdir.value
+    if working_directory is None:
+        working_directory = "."
+
+    relpath = parse_relative_directory(working_directory, field_set.address)
+    boot_script = f"cd {shlex.quote(relpath)}; " if relpath != "" else ""
+
+    return RunRequest(
+        digest=execution_environment.digest,
+        args=(
+            bash.path,
+            "-c",
+            boot_script + command,
+            f"{bin_name()} run {field_set.address.spec} --",
+        ),
+        extra_env=env_vars,
+        immutable_input_digests=FrozenDict.frozen(merged_extras.immutable_input_digests),
+        append_only_caches=FrozenDict.frozen(merged_extras.append_only_caches),
+    )
+
+
 def rules():
     return [
         *collect_rules(),
         *adhoc_process_support_rules(),
         UnionRule(GenerateSourcesRequest, GenerateFilesFromShellCommandRequest),
         *RunShellCommand.rules(),
+        *RunShellCommandBuild.rules(),
     ]