always use pex thru the cli, never as a python module (#22979)

The stable interface for using Pex is the cli, not as a Python module.
We removed most of these in #21515 but a few test dependencies remained.
Removing the last of those allows us to get Pex out of the lockfile and
make updating Pex versions smoother.

AI Notice: I used Claude to find the PexPEX based approach. This looks
reasonable to me but does not have a ton of idiomatic examples.

Author: cburroughs

3rdparty/python/requirements.txt

@@ -12,7 +12,6 @@ hdrhistogram==0.10.3
 ijson==3.4.0.post0
 libcst==1.8.5
 packaging==25.0
-pex==2.73.1
 psutil==5.9.8
 # This should be compatible with pytest.py, although it can be looser so that we don't
 # over-constrain pantsbuild.pants.testutil

3rdparty/python/user_reqs.lock

@@ -971,27 +971,6 @@
           "requires_python": ">=2.6",
           "version": "7.0.3"
         },
-        {
-          "artifacts": [
-            {
-              "algorithm": "sha256",
-              "hash": "dd6524d75476f0a0dd798f64c273e0328fd885d06e0e23aa9424d880f3bb0b4a",
-              "url": "https://files.pythonhosted.org/packages/8a/6e/b88fe4e39ce3909e10c53ac5c643b4f266fe802c68b80ae7104bff964518/pex-2.73.1-py2.py3-none-any.whl"
-            },
-            {
-              "algorithm": "sha256",
-              "hash": "334aaa83eb3762c8a32ec6dde5525cf1c6842589e307bdc65406e42c113c4e64",
-              "url": "https://files.pythonhosted.org/packages/bd/65/69761bac2c2bac5f21b94c3debac795ec924c45678c63dc5fe429189d550/pex-2.73.1.tar.gz"
-            }
-          ],
-          "project_name": "pex",
-          "requires_dists": [
-            "psutil>=5.3; extra == \"management\"",
-            "subprocess32>=3.2.7; python_version < \"3\" and extra == \"subprocess\""
-          ],
-          "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<3.16,>=2.7",
-          "version": "2.73.1"
-        },
         {
           "artifacts": [
             {
@@ -2255,7 +2234,6 @@
     "mypy-typing-asserts==0.1.1",
     "node-semver==0.9.0",
     "packaging==25.0",
-    "pex==2.73.1",
     "psutil==5.9.8",
     "pydevd-pycharm==251.23536.40",
     "pytest!=7.1.0,!=7.1.1,<9,>=7",

3rdparty/python/user_reqs.lock.metadata

@@ -20,7 +20,6 @@
     "mypy-typing-asserts==0.1.1",
     "node-semver==0.9.0",
     "packaging==25.0",
-    "pex==2.73.1",
     "psutil==5.9.8",
     "pydevd-pycharm==251.23536.40",
     "pytest!=7.1.0,!=7.1.1,<9,>=7",
@@ -47,4 +46,4 @@
   "overrides": [],
   "sources": [],
   "description": "This lockfile was generated by Pants. To regenerate, run: pants generate-lockfiles --resolve=python-default"
-}
+}

docs/notes/2.31.x.md

@@ -107,6 +107,8 @@ Pants no longer supports loading `pkg_resources`-style namespace packages for pl
 
 Allow `InteractiveProcess` to set the working directory relative to the sandbox or workspace. Existing usages of `InteractiveProcess.from_process` will now respect the working directory if its set on the Process, which may be a breaking change depending on the use case. Two existing rules `twine_upload` for python package uploads and `test_shell_command_interactively` for shell command testing with the `--debug` flag will now honor the working directory if set on the Process.
 
+Pex is no longer included *as a Python module* in the default lockfile.  Pex is still used as a cli tool as intended.
+
 #### nFPM backend
 
 Added a new rule to help in-repo plugins implement the `inject_nfpm_package_fields(InjectNfpmPackageFieldsRequest) -> InjectedNfpmPackageFields` polymorphic rule. The `get_package_field_sets_for_nfpm_content_file_deps` rule (in the `pants.backend.nfpm.util_rules.contents` module) collects selected `PackageFieldSet`s from the contents of an `nfpm_*_package` so that the packages can be analyzed to inject things like package requirements.

src/python/pants/backend/python/util_rules/BUILD

@@ -13,7 +13,7 @@ python_tests(
     name="tests",
     overrides={
         "local_dists_test.py": {"timeout": 120},
-        "pex_from_targets_test.py": {"timeout": 200, "dependencies": ["3rdparty/python#pex"]},
+        "pex_from_targets_test.py": {"timeout": 200},
         "pex_test.py": {"timeout": 600, "dependencies": [":complete_platform_pex_test"]},
         "package_dists_test.py": {"timeout": 150},
         "vcs_versioning_test.py": {"timeout": 120},

src/python/pants/backend/python/util_rules/pex_from_targets_test.py

@@ -4,8 +4,8 @@
 from __future__ import annotations
 
 import importlib.resources
+import os
 import subprocess
-import sys
 from collections.abc import Iterable
 from dataclasses import dataclass
 from enum import Enum
@@ -40,6 +40,7 @@
     PexRequest,
     PexRequirementsInfo,
 )
+from pants.backend.python.util_rules.pex_cli import PexPEX
 from pants.backend.python.util_rules.pex_from_targets import (
     ChosenPythonResolve,
     ChosenPythonResolveRequest,
@@ -493,37 +494,41 @@ def create_project_dir(workdir: Path, project: Project) -> PurePath:
     return project_dir
 
 
-def create_dists(workdir: Path, project: Project, *projects: Project) -> PurePath:
+def create_dists(
+    workdir: Path, rule_runner: PythonRuleRunner, project: Project, *projects: Project
+) -> PurePath:
     project_dirs = [create_project_dir(workdir, proj) for proj in (project, *projects)]
 
-    pex = workdir / "pex"
+    # Get the pex CLI binary and materialize it
+    pex_pex = rule_runner.request(PexPEX, [])
+    rule_runner.scheduler.write_digest(pex_pex.digest)
+    pex_binary = Path(rule_runner.build_root) / pex_pex.exe
+
+    pex_output = workdir / "output.pex"
     subprocess.run(
         args=[
-            sys.executable,
-            "-m",
-            "pex",
+            pex_binary,
             *project_dirs,
             *build_deps,
             "--include-tools",
             "-o",
-            pex,
+            pex_output,
         ],
         check=True,
     )
 
     find_links = workdir / "find-links"
     subprocess.run(
         args=[
-            sys.executable,
-            "-m",
-            "pex.tools",
-            pex,
+            pex_binary,
+            pex_output,
             "repository",
             "extract",
             "--find-links",
             find_links,
         ],
         check=True,
+        env=os.environ.copy() | {"PEX_MODULE": "pex.tools"},
     )
     return find_links
 
@@ -537,6 +542,7 @@ def test_constraints_validation(tmp_path: Path, rule_runner: PythonRuleRunner) -
     sdists.mkdir()
     find_links = create_dists(
         sdists,
+        rule_runner,
         Project("Foo-Bar", "1.0.0"),
         Project("Bar", "5.5.5"),
         Project("baz", "2.2.2"),
@@ -696,7 +702,7 @@ def test_exclude_requirements(
 ) -> None:
     sdists = tmp_path / "sdists"
     sdists.mkdir()
-    find_links = create_dists(sdists, Project("baz", "2.2.2"))
+    find_links = create_dists(sdists, rule_runner, Project("baz", "2.2.2"))
 
     rule_runner.write_files(
         {