diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 669c8fc6b1..17c2535f5a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -202,6 +202,63 @@ jobs: run: | pipenv run pytest -ra -n auto -v --fulltrace tests + tests-pure-python-backend: + # Initiative G phase 3b (T_CI1): run the full pipenv test suite + # under ``PIPENV_RESOLVER=pure-python`` to dogfood the new + # pure-Python resolver backend. Non-blocking + # (``continue-on-error: true``) — the acceptance gate is "matrix + # runs and produces a result", not "matrix passes", until the + # backend reaches feature parity with pip. Ubuntu + 3.12 only at + # this phase to keep CI minutes reasonable; widen the matrix once + # the backend is consistently green. + name: Tests (pure-python backend) / Ubuntu / 3.12 + needs: [tests-smoke] + runs-on: ubuntu-latest + continue-on-error: true + timeout-minutes: 45 + steps: + - uses: actions/checkout@v5 + - name: Set up Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install latest pip, setuptools, wheel + run: | + python -m pip install --upgrade pip setuptools wheel + - name: Install dependencies + env: + PIPENV_DEFAULT_PYTHON_VERSION: "3.12" + PYTHONWARNINGS: ignore:DEPRECATION + PYTHONIOENCODING: "utf-8" + GIT_ASK_YESNO: "false" + run: | + git submodule sync + git submodule update --init --recursive + python -m pip install -e . --upgrade + pipenv install --deploy --dev --python=3.12 + pipenv run python -m pip install -e ".[tests,dev]" --upgrade + - name: Run pypiserver with pipenv + run: | + pipenv run pypi-server --version + pipenv run pypi-server run -v --host=0.0.0.0 --port=8080 --hash-algo=sha256 --disable-fallback --welcome /dev/null ./tests/pypi/ ./tests/fixtures & + - name: Run tests with pure-python backend + env: + PIPENV_DEFAULT_PYTHON_VERSION: "3.12" + PYTHONWARNINGS: ignore:DEPRECATION + PIPENV_NOSPIN: "1" + CI: "1" + GIT_ASK_YESNO: "false" + PYPI_VENDOR_DIR: "./tests/pypi/" + PYTHONIOENCODING: "utf-8" + GIT_SSH_COMMAND: ssh -o StrictHostKeyChecking=accept-new -o CheckHostIP=no + # T_CI1 (Initiative G phase 3b): select the pure-Python + # backend via the documented env-var path. Honoured by + # ``pipenv.resolver.core._selected_backend_name`` after the + # parent-side CLI / Pipfile chain falls through. + PIPENV_RESOLVER: pure-python + run: | + pipenv run pytest -ra -n auto -v --fulltrace tests + resolver-module-coverage: # Initiative G phase 1 (T17): enforce per-module coverage floors # for the new pure-Python simple-API client surface. Without this diff --git a/docs/dev/initiative-g-phase3-design.md b/docs/dev/initiative-g-phase3-design.md new file mode 100644 index 0000000000..ee00ba78c9 --- /dev/null +++ b/docs/dev/initiative-g-phase3-design.md @@ -0,0 +1,545 @@ +# Initiative G — Phase 3 Design: Pure-Python `resolvelib.Provider` Backend + +Status: **draft awaiting maintainer sign-off**. No code change under +Phase 3 until this document is approved. + +Companion documents: + +- [`initiative-g-pure-python-design.md`](./initiative-g-pure-python-design.md) — + the umbrella design doc. Phase 3's scope was sketched there at §5.4 + and §7.4; this doc fills in the detail. +- [`initiative-f-backends-design.md`](./initiative-f-backends-design.md) — + the `Backend` protocol Phase 3's deliverable plugs into. +- [`../../initiative-g-phase1-2-plan.md`](../../initiative-g-phase1-2-plan.md) — + Phases 1 + 2 swarm plan (shipped). Phase 3 has its own plan at + `../../initiative-g-phase3-plan.md`. + +## Table of contents + +1. [Summary](#1-summary) +2. [Motivation](#2-motivation) +3. [Scope](#3-scope) +4. [Architecture overview](#4-architecture-overview) +5. [Component design](#5-component-design) +6. [Migration path](#6-migration-path) +7. [Open questions for maintainer sign-off](#7-open-questions-for-maintainer-sign-off) +8. [Acceptance criteria](#8-acceptance-criteria) +9. [Out of scope](#9-out-of-scope) + +--- + +## 1. Summary + +Phase 3 implements the in-tree pure-Python `resolvelib.Provider` adapter +that consumes the typed `Candidate`s from `pipenv.resolver.pep691` / +`pipenv.resolver.manifest_cache` / `pipenv.resolver.fetcher` (all +shipped in Phases 1 + 2) and drives `resolvelib.Resolver` directly — +**without going through pip's `PackageFinder`, `LinkEvaluator`, or +`InstallRequirement`**. + +The result is a new selectable backend `pure-python` +(`pipenv/resolver/backends/pure_python.py`) under the Initiative F +backend registry. pip remains the default; users opt in via +`[pipenv] resolver_backend = "pure-python"` (or `--backend pure-python`). +Phase 4 (separate sign-off, post-production data) promotes to default +and eventually removes the pip backend. + +The post-`cf53eb17` CI baseline is `lock-warm ≈ 17 s` on the 100-pkg +bench. Phase 3's target is **≥ 30 % faster than the pre-phase-5 +baseline** (i.e., lock-warm ≤ 14.5 s) per §8 below. + +## 2. Motivation + +Phase 5's perf hunt brought CI lock-warm from ~21 s to ~17 s +(~21 % — `cf53eb17` did most of that). The remaining ~17 s is +architecturally bounded by pip's internals: + +| where the time goes (post-`cf53eb17`) | wall | category | +| --------------------------------------------------- | ------- | -------- | +| pip's resolvelib backtracking loop | ~10 s | pip internal | +| `Link.from_json` × 107 k (~99 k unique) | ~3.3 s | pip internal — Link memoization investigated and dropped (only 7 % cache hits) | +| `evaluate_link` × 107 k | ~2.2 s | pip internal | +| `_ensure_quoted_url` × 107 k | ~2.2 s | pip internal | +| `packaging.version.__str__` × 455 k | ~1 s | pip internal (via `canonicalize_version` + resolvelib reporter) | +| network I/O (urllib3 + ssl + zlib) | ~5 s | network ceiling | + +Every row except the network ceiling is **pip's own machinery +operating on its own data types**. We can't move it without replacing +the data path. Phase 3 replaces it: + +- Our `Candidate` (typed dataclass) replaces pip's `Link` (mutable + + URL-canonicalization-per-construction). +- Our `ParsedManifestCache` (per-package parsed list on disk) replaces + pip's raw-response cache + per-backtrack re-parse. +- Our `Provider` drives `resolvelib.Resolver` directly with typed + `Candidate`s — no `LinkEvaluator`, no `Wheel` reconstruction per + candidate, no `canonicalize_version` stringification storm. + +This is the only remaining lever short of switching off pipenv-on-pip +entirely. + +## 3. Scope + +### 3.1 In scope + +- A `Requirement` dataclass — frozen, hashable, carrying name, + specifier, extras, marker, source (Pipfile / transitive / etc.). +- A `MetadataFetcher` that reads wheel `METADATA` via PEP 658 when the + index advertises it, falling back to a partial-download path (HEAD + + range request for the zip directory) for indexes that don't. +- A `PurePythonProvider` implementing `resolvelib.AbstractProvider` + (`identify`, `get_preference`, `find_matches`, `is_satisfied_by`, + `get_dependencies`) over `Candidate`. +- A `PurePythonBackend` (Initiative F `Backend` protocol impl) + translating `ResolverRequest` → resolvelib resolve → `ResolverResponse`. +- Backend registration at `pipenv/resolver/backends/__init__.py`. +- Lockfile-parity test suite against the pip backend. + +### 3.2 Out of scope + +- **sdist handling**. sdists need `pyproject.toml` / `setup.py + egg_info` build to extract metadata. Phase 3 ships **wheel-only + resolves**; an sdist appearing in the candidate set causes + `PurePythonBackend.resolve` to fall back to the pip backend for + that resolution (or fail loud if the user opted out — see Q-A). + Native sdist support is Phase 4. +- **Removing the pip backend**. Phase 4. +- **HTTP/2 transport**. Still HTTP/1.1 with thread-pool parallelism + per Phase 1's decision. HTTP/2 is a separate post-Phase-3 effort. +- **Cross-platform lockfiles**. Same as Phase 1: orthogonal. +- **Custom resolvers**. We continue to drive `resolvelib`; replacing + the resolver itself is not on the roadmap. + +--- + +## 4. Architecture overview + +``` + ┌──────────────────────────────────────────┐ + │ pipenv parent process │ + │ do_lock / do_install │ + │ │ ResolverRequest (Initiative F) │ + └──────┼──────────────────────────────────────┘ + │ + ┌────────────┴────────────┐ + │ │ + pip backend pure-python backend ← Phase 3 (NEW) + (existing) │ + ├── PurePythonProvider + │ (resolvelib.AbstractProvider) + │ │ + │ │ candidates from + │ │ ParsedManifestCache + │ │ + │ ▼ + │ PEP691Client + ParallelFetcher + │ │ (Phases 1 + 2 infra) + │ │ + │ ▼ + │ ParsedManifestCache + │ (on-disk JSON) + │ + ├── MetadataFetcher + │ (PEP 658 + wheel-head fallback) + │ + ├── Requirement (typed dataclass) + │ + ▼ + resolvelib.Resolver + │ + ▼ resolved graph + ResolverResponse (typed) +``` + +Key points: + +- The `Backend` protocol is the only shared seam between the parent + process and the resolver implementation. Same as pip backend; same + `ResolverRequest` / `ResolverResponse` envelopes (Initiative F). +- `PurePythonProvider` is a `resolvelib.AbstractProvider` subclass; it + doesn't subclass pip's `PipProvider`. This deliberately gives us + zero `pip._internal.*` imports in the new code path (enforced by + Phase 1's pre-commit grep gate, scoped to `pipenv/resolver/`). +- `MetadataFetcher` is the only component that may need a per-package + network round-trip *during* resolution (when an index doesn't + advertise PEP 658 metadata). All other candidate data comes from + the pre-warmed `ParsedManifestCache`. + +--- + +## 5. Component design + +### 5.1 `Requirement` — typed requirement model + +**Module**: `pipenv/resolver/pure_python_requirement.py` + +Frozen dataclass — represents one constraint in the resolution graph: + +```python +@dataclass(frozen=True, slots=True) +class Requirement: + name: str # PEP 503 canonical + specifier: SpecifierSet # may be empty (any version) + extras: frozenset[str] # empty if no extras + marker: Marker | None # environment marker + source: Literal["pipfile", "transitive", "constraint"] + parent: str | None = None # name of the candidate that + # produced this transitive +``` + +`SpecifierSet` and `Marker` come from `pipenv.vendor.packaging` +(already vendored, not pip-internal). + +`identify(requirement)` returns `(name, frozenset(extras))` for +`resolvelib`'s key — matches pip's grouping where requests for the +same package with different extras are treated as separate identifiers +in the dependency graph. + +### 5.2 `MetadataFetcher` — PEP 658 + wheel-head fallback + +**Module**: `pipenv/resolver/pure_python_metadata.py` + +Responsibility: given a `Candidate`, return its parsed core metadata +(name, version, requires-dist list, requires-python, etc.). + +**Two-tier strategy**: + +1. **PEP 658 fast path** (`Candidate.has_pep658_metadata` — added to + `Candidate` in Phase 1's design, deferred per Q2; this design + answers Q2: yes, plumb it through, see §7). When the index + advertised `core-metadata`, fetch the `.metadata` file + directly. Verify against the advertised hash. Parse via stdlib + `email.parser`. Cost: one small HTTP GET per wheel (~3-5 kB + typical). + +2. **Wheel-head fallback** for indexes that don't advertise PEP 658 + (or for cases where the `.metadata` file is missing). Use HTTP + Range requests to download just the central-directory portion of + the wheel (the last ~few kB), parse the central directory to + locate `METADATA`, then fetch a second range covering just that + entry. Falls back to the full wheel download as a last resort. + +**Caching**: a small `MetadataCache` on disk +(`/pipenv-manifests/metadata-v1/`) keyed by +`(wheel_url, content_hash)`. Resolves are append-only; cache entries +are valid forever (wheels are immutable on PyPI). + +### 5.3 `PurePythonProvider` — `resolvelib.AbstractProvider` + +**Module**: `pipenv/resolver/pure_python_provider.py` + +Implements the five `AbstractProvider` methods: + +- **`identify(req_or_cand)`**: returns `(canonical_name, + frozenset(extras))` for both `Requirement` and `Candidate`. + +- **`get_preference(identifier, resolutions, candidates, information, + backtrack_causes)`**: returns a small tuple that drives resolvelib's + ordering. **Mirror pip's pip-internal `provider.py:get_preference` + exactly** — this is the parity-critical method. Specifically: + - Pinned-version requirements get the highest preference (lowest + sort key). + - Requirements from Pipfile beat transitive requirements. + - Backtrack-causing identifiers get re-tried later. + +- **`find_matches(identifier, requirements, incompatibilities)`**: + the hot path. Reads the `ParsedManifestCache` for the + `(index_url, name)` tuple, filters to candidates satisfying every + `requirement` and not in `incompatibilities`, returns an iterator + ordered highest-version-first (per the version comparison rules + in `packaging.version.Version`). No network I/O — assumes the + cache was warmed pre-resolve by `ParallelFetcher`. On cache miss + (transitive dep we didn't pre-fetch) — lazy network fetch. + +- **`is_satisfied_by(requirement, candidate)`**: `candidate.version + in requirement.specifier` AND extras compatibility AND + marker-evaluates-True against the target environment. + +- **`get_dependencies(candidate)`**: invokes `MetadataFetcher.fetch` + for wheel candidates. Parses `Requires-Dist` into `Requirement` + list. For sdist candidates: see Q-A. + +### 5.4 `PurePythonBackend` — `Backend` protocol impl + +**Module**: `pipenv/resolver/backends/pure_python.py` + +```python +class PurePythonBackend: + name = "pure-python" + + def is_available(self) -> bool: + return True # in-tree; always available + + def resolve(self, request: ResolverRequest) -> ResolverResponse: + # 1. Pre-fetch top-level package indexes (Phase 2 infra). + # 2. Build Requirement objects from request.packages.specs. + # 3. Build PurePythonProvider against the cache + fetcher. + # 4. Drive resolvelib.Resolver to a fixed point. + # 5. Translate resolved graph -> ResolverResponse. +``` + +Failure modes (each maps to a structured `ResolutionError` / +`InternalError` per the `Backend` protocol contract): + +- `resolvelib.ResolutionImpossible` → `ResolutionError` with the + conflict list translated into `ResolverResponse.conflicts`. +- Network failure during `get_dependencies` for a wheel that's + required → `InternalError` with the URL redacted. +- sdist appears in the candidate path → see Q-A. + +--- + +## 6. Migration path + +### Phase 3.1 — `Requirement` + `MetadataFetcher` standalone + +**Deliverable**: `pure_python_requirement.py`, `pure_python_metadata.py` ++ unit tests. No `Provider`, no `Backend`, no integration. + +**Acceptance**: round-trip + standard parsing on a fixture set of 20 +PyPI wheels (PEP 658 + non-PEP-658 split). + +**Scope estimate**: ~1 week. + +### Phase 3.2 — `PurePythonProvider` + +**Deliverable**: full `resolvelib.AbstractProvider` impl + unit tests. +No `Backend` integration; tests drive `resolvelib.Resolver` directly +against a mock `ParsedManifestCache`. + +**Acceptance**: resolves the 100-pkg bench Pipfile to the same +graph as pip backend would. Per-package and final-graph diff in a +companion `parity-matrix.md` doc. + +**Scope estimate**: 2-3 weeks. + +### Phase 3.3 — `PurePythonBackend` + registry wiring + +**Deliverable**: backend selectable via `[pipenv] resolver_backend += "pure-python"`. Lock + install end-to-end. Sandboxed under a +matrix CI step against pip versions N-1, N, N+1. + +**Acceptance**: lockfile byte-identical (modulo field ordering) to pip +backend for the bench fixture + 10 real-world projects (curated list +in the plan). + +**Scope estimate**: 1-2 weeks. + +### Phase 3.4 — Bench + parity matrix doc + +**Deliverable**: published numbers from the CI bench under both +backends. Parity matrix doc enumerates every behavioural divergence +with justification. + +**Acceptance**: lock-warm ≤ 14.5 s on the 100-pkg bench (≥ 30 % off +the pre-phase-5 baseline of 21.3 s); parity matrix has zero +unjustified entries. + +**Scope estimate**: ~1 week. + +--- + +## 7. Decisions for execution (maintainer sign-off 2026-05-12) + +The questions originally framed as "open" here have been answered. +Decisions below are load-bearing for the plan in +[`../../initiative-g-phase3-plan.md`](../../initiative-g-phase3-plan.md); +flipping any of them requires re-planning. + +**Q-A: sdist handling → FAIL LOUD** + +When the pure-python backend encounters an sdist-only candidate (no +wheel for the target Python + platform), the backend raises a typed +`InternalError` naming the trigger. The user adjusts the Pipfile or +explicitly switches `[pipenv] resolver_backend = "pip"` for that +resolve. No silent transparent fallback to pip backend. + +Rationale: behaviour is auditable; users always know which backend +their lockfile came from; no "this worked in my CI yesterday and +silently fell back today" surprises. Combined with Q-F's pre-check, +the failure surfaces at lock-startup rather than deep in a 30-second +resolve. + +Implementation: T9 raises typed `_SdistEncountered` from +`get_dependencies`; `PurePythonBackend.resolve` translates that into +`ResolverResponse.result = InternalError(message=...)`. + +--- + +**Q-B: PEP 658 metadata pre-fetch → TOP-LEVEL ONLY** + +`ParallelFetcher` pre-fetches `core-metadata` files concurrently with +the simple-API JSON for top-level Pipfile packages. Transitive +candidates stay lazy — their metadata is fetched during +`get_dependencies`. + +Rationale: bounded bandwidth (~500 kB at 100 packages) with guaranteed +parallelism on the most-likely-hit set. + +--- + +**Q-C: `get_preference` parity → STRICT MIRROR** + +`PurePythonProvider.get_preference` exactly mirrors pip's +`pipenv/patched/pip/_internal/resolution/resolvelib/provider.py:176` +tuple shape and tie-breaking. Lockfile-byte-identity for the +T15 bench + T_PARITY_REAL real-world list is the gate. + +Rationale: any user-visible divergence is a bug (or a deliberate doc +entry in T_PARITY_MATRIX); maintainers can audit the matrix +post-merge. + +--- + +**Q-D: keyring auth → NOT IN PHASE 3** + +netrc + URL-embedded basic-auth + `PIP_CLIENT_CERT` are the only +supported auth backends for the new path. Keyring users stay on the +pip backend. Phase 4 can scope this separately based on bug-report +data once pure-python has real users. + +Rationale: keyring touches user keychains; failure modes are subtle; +no real signal yet on which keyring providers our users need. + +--- + +**Q-E: T_PARITY_REAL real-world list → 10 WHEEL-HEAVY PROJECTS** + +The parity test exercises these 10 mainstream wheel-heavy +combinations: + +1. `django` + `psycopg[binary]` +2. `flask` + `gunicorn` +3. `fastapi` + `uvicorn` +4. `requests` + `httpx` +5. `pandas` + `numpy` +6. `pytest` + `pytest-cov` +7. `sqlalchemy` + `alembic` +8. `cryptography` + `pyopenssl` +9. `boto3` + `botocore` +10. `click` + `rich` + +Rationale: chosen to avoid known sdist-only transitives so the Q-A +fail-loud path doesn't dominate the parity gate. If any entry develops +sdist-only deps over time (a Python-version-N+1 wheel hasn't shipped +yet, etc.), T_PARITY_REAL marks that entry pending; the gate stays +green on the other 9. + +--- + +**Q-F: sdist UX → BACKEND-STARTUP PRE-CHECK** + +When `resolver_backend = "pure-python"` is active, `PurePythonBackend.resolve` +runs a fast pre-check on the top-level Pipfile packages immediately +after `ParallelFetcher.populate` returns: + +- For each top-level package, iterate its candidates. +- If at least one candidate is a wheel compatible with the target + Python + platform → OK. +- Otherwise: raise `ResolutionError` with a message naming the + offending package(s) and pointing at either pinning to a version + with wheels or `pipenv lock --backend pip` for this resolve. + +This catches the common case (a top-level Pipfile entry whose only +release is an sdist) at lock-startup, before resolvelib spends 30 +seconds chasing a transitive graph that's going to fail anyway. +Transitive sdist-only candidates still hit the deeper Q-A fail-loud +path inside `get_dependencies` (rarer; harder to surface early). + +Rationale: keep the error close to the cause. No "different backend" +suggestion phrasing — just `pipenv lock --backend pip` if the user +chooses to retry that way. + +--- + +## 8. Acceptance criteria + +Phase 3 ships when: + +- All Phase 3 plan tasks are complete. +- `pipenv/resolver/backends/pure_python.py` + its dependencies (`Provider`, + `Requirement`, `MetadataFetcher`) import zero `pip._internal.*` + symbols (enforced by Phase 1's pre-commit gate). +- Lockfile byte-identity vs pip backend across: + - The 100-package bench fixture. + - The 10-real-PyPI-project list defined in the plan. + - pip versions N-1, N, N+1 in the matrix CI step. +- Parity matrix doc shipped with every divergence justified. +- `lock-warm` on CI ≤ 14.5 s (≥ 30 % off the pre-phase-5 baseline of + 21.3 s — the Phase 3 perf gate). +- News fragment + user-facing docs for the new `resolver_backend` + selector. + +### Phase 3b — sdist + markers + CI dogfood (maintainer sign-off 2026-05-12) + +Phase 3a's static parity gate (byte-identity on a 100-package bench) +is preserved as a sub-criterion; the **strong gate for Phase 3b is +CI dogfood** — the entire pipenv test suite must run under the +new backend and produce a result on every PR. + +| Gate | Source | Blocking? | Status (2026-05-12) | +| ------------------------------- | ----------------------- | --------- | ------------------- | +| Resolver-module unit coverage ≥ 90 % | `resolver-module-coverage` CI job | Yes | Green (98 % aggregate post-T_S6) | +| Lockfile byte-identity on `click=*` smoke | Local smoke (Phase 3a) | Yes (regression-tripwire) | Green (post-T_S5) | +| Lockfile byte-identity on T15 100-package bench | `tests/integration` benchmark | Sub-criterion of CI dogfood | Pending bench refresh | +| Full test suite under `PIPENV_RESOLVER=pure-python` | `tests-pure-python-backend` CI job (T_CI1) | **No (non-blocking)** until consistently green | Wired (T_CI1, 2026-05-12) | + +The `tests-pure-python-backend` job carries `continue-on-error: true`. +It is a **dogfood gate**, not a release gate, until the backend +demonstrates a clean run for ≥ 2 consecutive weeks of PR traffic. At +that point the flag is flipped (separate one-line PR) and the gate +becomes blocking. + +**Q-A flip (Phase 3a → Phase 3b)**: Q-A originally locked sdist +handling at "fail loud" — `_SdistEncountered` raised by +`get_dependencies` and translated to `InternalError`. Phase 3b +**flips this to "build transparently via PEP 517"**: the backend +now resolves sdist-only candidates by downloading, extracting, and +invoking the project's PEP 517 build backend (or +`setuptools.build_meta:__legacy__` fallback) to recover `METADATA`. +The transparent build is implemented in +`pipenv/resolver/pure_python_sdist.py` and routed via +`pipenv/resolver/pure_python_metadata.py::fetch_metadata`'s `.tar.*` +/ `.zip` branch (T_S1, T_S2). + +The Q-F backend-startup pre-check survives in repurposed form: it +no longer flags sdists (those now resolve), but it still fires when +a top-level package has **zero** candidates of any kind, surfacing +a "no distfiles available" error at lock-startup rather than mid-resolve +(T_S4). + +**Build isolation**: the sdist build path uses +:class:`build.env.DefaultIsolatedEnv` to create a throwaway venv for +each sdist's PEP 517 hooks. This matches pip's `--use-pep517` default +and ensures that package-specific build backends (poetry-core, +hatchling, flit-core, ...) do not crash on import when they are not +installed in pipenv's own interpreter. + +The isolated environment: + +- Installs `[build-system].requires` plus + `get_requires_for_build_wheel` dependencies into a fresh venv + before invoking the backend's `prepare_metadata_for_build_wheel` + hook. +- Runs in a temp directory; the cache stores the extracted `METADATA` + only, never the build sandbox. +- Includes path-traversal validation on every tar / zip member name + before extraction (`pure_python_sdist._validate_member_name`). +- Hard-rejects device / symlink / fifo members. +- Is wrapped in a 300-second timeout to prevent wedged build backends + from blocking resolution indefinitely. + +Users who prefer pip's resolver can still pin +`resolver_backend = "pip"` for that resolve. + +## 9. Out of scope + +- ~~sdist resolution (Phase 4).~~ Resolved in Phase 3b via PEP 517 + build (see §8 Phase 3b row + Q-A flip). +- Removing pip backend (Phase 4). +- HTTP/2 transport. +- Replacing `resolvelib`. +- Cross-platform lockfiles. +- Keyring auth (deferred per Q-D above). +- Isolated-environment sdist builds (Phase 4 — see §8 Phase 3b + "no-build-isolation tradeoff"). diff --git a/initiative-g-phase3-plan.md b/initiative-g-phase3-plan.md new file mode 100644 index 0000000000..c634e61476 --- /dev/null +++ b/initiative-g-phase3-plan.md @@ -0,0 +1,1378 @@ +# Plan: Initiative G — Phase 3 (Pure-Python `resolvelib.Provider` Backend) + +**Generated**: 2026-05-12 +**Source design doc**: [`docs/dev/initiative-g-phase3-design.md`](docs/dev/initiative-g-phase3-design.md) +**Branch**: `maintenance/code-cleanup-phase6-pure-python-resolver-2026-07` (off +`maintenance/code-cleanup-phase5-perf-2026-06` at commit `3d16ca04`). +**Scope**: Phase 3 only. Phase 4 (promote to default + remove pip +backend) is a separate sign-off after Phase 3 ships and a release +cycle of opt-in user data lands. + +--- + +## Overview + +Phase 3 implements the in-tree `pure-python` resolver backend that +replaces pip's `PackageFinder` + `LinkEvaluator` + `InstallRequirement` +machinery with our own `Requirement` + `MetadataFetcher` + +`PurePythonProvider` chain over `resolvelib.AbstractProvider`. + +The Phases 1 + 2 infrastructure already shipped (`PEP691Client`, +`ParsedManifestCache`, `ParallelFetcher`, `Candidate`, `Hash`, etc.) — +this plan picks them up unchanged. + +### Acceptance criteria (from design doc §8) + +**Phase 3:** +- Zero `pip._internal.*` imports in `pipenv/resolver/pure_python_*.py` + and `pipenv/resolver/backends/pure_python.py` (enforced by Phase 1's + pre-commit grep gate). +- Lockfile byte-identity vs pip backend on: + - The 100-package bench fixture. + - The 10-real-PyPI-project list defined in T_PARITY_REAL below. + - pip versions N-1, N, N+1. +- Parity matrix doc shipped with every divergence justified. +- CI `lock-warm` ≤ 14.5 s on the 100-pkg bench (≥ 30 % off the + pre-phase-5 baseline of 21.3 s). +- News fragment + user-facing docs for `[pipenv] resolver_backend = + "pure-python"`. + +### Decisions baked into this plan (sign-off 2026-05-12) + +The design doc §7 catalogues six decisions (Q-A through Q-F). All are +now answered; this plan reflects them load-bearingly. + +- **Q-A** sdist handling → **fail loud** (transparent fallback was + considered and rejected; behaviour must be auditable). +- **Q-B** PEP 658 pre-fetch → **top-level packages only**, transitives + lazy. +- **Q-C** `get_preference` parity → **strict mirror with + byte-identity gate**. +- **Q-D** keyring auth → **not in Phase 3** (netrc + URL-auth + + `PIP_CLIENT_CERT` only). +- **Q-E** parity-real list → **10 wheel-heavy projects** (django + + psycopg[binary], flask + gunicorn, fastapi + uvicorn, requests + + httpx, pandas + numpy, pytest + pytest-cov, sqlalchemy + alembic, + cryptography + pyopenssl, boto3 + botocore, click + rich). +- **Q-F** sdist UX → **backend-startup pre-check** on top-level + packages with a clear error message; transitive sdists still fail + loud per Q-A but rarer. + +--- + +## Prerequisites + +- Branch `maintenance/code-cleanup-phase6-pure-python-resolver-2026-07` + exists and is checked out. +- Initiative F backend registry is at `pipenv/resolver/backends/` with + `base.py` + `pip.py` + `__init__.py`. +- Phase 1 + 2 modules at `pipenv/resolver/` (candidate, pep691, + pep691_types, manifest_cache, fetcher, auth) are unchanged. +- Vendored `pipenv.vendor.packaging` provides `Version`, `SpecifierSet`, + `Marker`, `Requirement` (as a parser, not the type we'll use for + graph nodes), `Tag`, `parse_wheel_filename`. +- Vendored `pipenv.patched.pip._vendor.resolvelib` provides + `AbstractProvider`, `Resolver`, `ResolutionImpossible`. + +--- + +## Dependency Graph (high-level) + +``` + T1 ─┬──────────────────────────────┐ + │ │ + T2 ─┤ │ + │ │ + T3 ── T4 ── T5 ── T6 ── T7 ── T8 + │ + T9 ── T10 ── T11 ── T12 ── T13 + │ + T14 +``` + +The canonical wave breakdown is in **§ Parallel Execution Groups** +near the bottom of this document. + +--- + +## Tasks + +### T1: `Requirement` dataclass + +- **depends_on**: `[]` +- **location**: `pipenv/resolver/pure_python_requirement.py` (new) +- **description**: + Frozen `@dataclass(frozen=True, slots=True)` per design §5.1. + Fields: `name` (PEP 503 canonical), `specifier` (`SpecifierSet`), + `extras` (`frozenset[str]`), `marker` (`Marker | None`), `source` + (Literal `"pipfile" | "transitive" | "constraint"`), `parent` (`str | + None`). Use `pipenv.vendor.packaging.specifiers.SpecifierSet` and + `pipenv.vendor.packaging.markers.Marker`. Add a + `Requirement.from_pipfile_entry(name, value)` helper that handles + the common Pipfile shapes (string version, dict with `version`/`extras`/`markers`, + `*` for any-version). +- **validation**: + - Round-trip: build a `Requirement` with all fields set; verify each + attribute; verify equality + hashability (frozenset membership). + - `from_pipfile_entry("django", "*")` → spec is empty SpecifierSet. + - `from_pipfile_entry("django", ">=4.0,<6")` → spec parses. + - `from_pipfile_entry("django", {"version": ">=4.0", "extras": ["argon2"]})` + → extras populated. + - Zero `pip._internal.*` imports. +- **status**: Completed +- **log**: + - 2026-05-12 — Implemented per design §5.1. Frozen + `@dataclass(frozen=True, slots=True)` with the six fields + enumerated in the brief (`name`, `specifier`, `extras`, `marker`, + `source`, `parent`). Hashability comes for free from + `frozen=True` — both `SpecifierSet` + (`pipenv/vendor/packaging/specifiers.py:842`) and `Marker` + (`pipenv/vendor/packaging/markers.py:329`) are hashable in the + vendored `packaging`, so no `__hash__` override is needed. The + `from_pipfile_entry` classmethod handles the four canonical + Pipfile shapes from the design brief: bare string version, + `"*"`, dict with `version`/`extras`/`markers`, and dict with + `"version": "*"`. Names are canonicalised via + `pipenv.vendor.packaging.utils.canonicalize_name` (PEP 503). + Unknown value types (neither str nor dict) raise `TypeError` so a + malformed Pipfile entry surfaces loudly rather than producing a + half-populated constraint. RED→GREEN with + `tests/unit/test_pure_python_requirement.py` (15 tests; all + pass). T11 will extend that file with the broader coverage + matrix (negative paths, edge cases). + `grep -n "pip\._internal" pipenv/resolver/pure_python_requirement.py` + shows zero matches; `ruff check` clean. +- **files edited/created**: + - `pipenv/resolver/pure_python_requirement.py` (replaced stub with + full implementation — `Requirement` dataclass + `from_pipfile_entry` + classmethod) + - `tests/unit/test_pure_python_requirement.py` (new; 15 tests + covering the four T1 acceptance criteria — round-trip / + equality / hashability / `from_pipfile_entry` shapes — plus the + `transitive` + `constraint` source-label and parent-propagation + cases T3/T7 rely on; T11 extends this file) + +--- + +### T2: `MetadataFetcher` — PEP 658 fast path + wheel-head fallback + +- **depends_on**: `[]` +- **location**: `pipenv/resolver/pure_python_metadata.py` (new) +- **description**: + Per design §5.2: + - `fetch_metadata(candidate: Candidate, session: PipSession) -> CoreMetadata`. + - **PEP 658 fast path**: when `candidate.metadata_url` is set (Phase + 1's `Candidate` already has `metadata_file_data`; widen if needed + per Q-B implementation), GET `.metadata`, verify the + advertised hash, parse via stdlib `email.parser.HeaderParser` + using `email.policy.compat32`. + - **Wheel-head fallback**: HTTP `HEAD` to get `Content-Length`, + then GET with `Range: bytes=-65536` (last 64 kB) to capture the + zip central directory. Parse the directory entries via + `zipfile.ZipFile(io.BytesIO(...))` — Python's stdlib zipfile + handles partial reads if the central directory is intact. Locate + the `/METADATA` entry, fetch a second range covering + just that entry, parse. + - **MetadataCache** on disk at `/pipenv-manifests/metadata-v1/` + keyed by `sha256(wheel_url)`. Cache entries are valid forever + (wheels are immutable). Atomic write same as `ParsedManifestCache`. + - `CoreMetadata` dataclass: `name`, `version`, `requires_python`, + `requires_dist` (list of strings), `provides_extras` (frozenset), + `summary` (for diagnostics). +- **validation**: + - PEP 658 fast path: against a real PyPI wheel that advertises + `core-metadata` (e.g., `numpy-1.26.0-*.whl`), fetch metadata and + assert `requires_dist` matches the known `Requires-Dist` headers. + - Wheel-head fallback: against a synthetic test wheel (built with + `tmp_path` + `zipfile`), fetch metadata via the range path and + assert it matches. + - Cache round-trip: `fetch_metadata` populates the cache; second call + on the same wheel returns from cache without network. + - Zero `pip._internal.*` imports. +- **status**: Completed +- **log**: + - 2026-05-12 — Implemented per design §5.2. RED→GREEN with + `tests/unit/test_pure_python_metadata.py` (9 tests; all pass). + PEP 658 fast path verifies advertised `sha256` against + `hashlib.sha256(body).hexdigest()`; mismatch raises + `MetadataFetchError`. Wheel-head fallback uses an `io.RawIOBase` + shim (`_PartialFile`) that lets `zipfile.ZipFile` see a + seekable view over the wheel while transparently re-issuing + HTTP range GETs for bytes outside the in-memory window — this + handles both the central-directory walk and the subsequent + `METADATA` extraction without buffering the whole wheel. HEAD + rejection (405/403) falls back to a probing `GET Range: bytes=0-1` + that reads the total length from `Content-Range`. Cache is keyed + by `sha256(wheel_url)` and uses the same tempfile + `os.replace` + atomic-write contract as `ParsedManifestCache`. Since Phase 1's + `Candidate` does not yet carry PEP 658 advertisement, + `fetch_metadata` accepts optional `metadata_url` + `metadata_hash` + keyword arguments — T3 / T7 can widen `Candidate` later and pass + the values through without API churn. `grep -nE + "^[[:space:]]*(from|import)[[:space:]]+pipenv\.patched\.pip\._internal" + pipenv/resolver/pure_python_metadata.py` shows zero matches. + `ruff check` clean. +- **files edited/created**: + - `pipenv/resolver/pure_python_metadata.py` (replaced stub with + full implementation — `CoreMetadata`, `MetadataFetchError`, + `MetadataCache`, `fetch_metadata`, `_parse_metadata_text`, + `_PartialFile`) + - `tests/unit/test_pure_python_metadata.py` (new; 9 tests covering + the four T2 acceptance criteria + helper coverage; T12 will + extend this file) + +--- + +### T3: `PurePythonProvider.identify` + +- **depends_on**: `[T1]` +- **location**: `pipenv/resolver/pure_python_provider.py` (new — start + the file with just `identify`; subsequent tasks fill it in) +- **description**: + Per design §5.3. Class shell: + ```python + class PurePythonProvider(AbstractProvider): + def __init__(self, *, cache, fetcher, metadata_fetcher, target_env): ... + + def identify(self, req_or_cand) -> tuple[str, frozenset[str]]: + """Group key: (canonical_name, frozenset(extras)).""" + ... + ``` + Accept both `Requirement` and `Candidate` instances; extract + `name` + `extras` consistently. `Candidate.extras` doesn't exist + yet (Phase 1 didn't need it) — add it as part of T3 if missing. +- **validation**: + - `identify(Requirement(name="Django", extras=frozenset({"argon2"}), ...))` + returns `("django", frozenset({"argon2"}))`. + - `identify(Candidate(name="django", ...))` returns + `("django", frozenset())`. + - Round-trip equality: two requirements with same name + extras + have equal `identify` outputs. +- **status**: Completed +- **log**: + - 2026-05-12 — Implemented per design §5.3. Started + `pipenv/resolver/pure_python_provider.py` with the + `PurePythonProvider(AbstractProvider)` shell: keyword-only + `__init__(*, cache, fetcher, metadata_fetcher, target_env)` stores + all four collaborators verbatim (typed `Any` for now — T4-T7 will + tighten as they consume each one), and `identify` returns the + `(canonical_name, frozenset(extras))` group key from design §5.3. + The `identify` body branches on `isinstance(_, Requirement)` for + static-checker narrowing on the `Requirement` path, and uses + duck-typing (`getattr(_, "extras", frozenset())`) on the + `Candidate` path so this module doesn't import + `pipenv.resolver.candidate` (smaller import graph; dodges any + future circular if T9's backend hands the provider a `Candidate` + subclass). `AbstractProvider` in + `pipenv/patched/pip/_vendor/resolvelib/providers.py` raises plain + `NotImplementedError` (no `@abstractmethod`), so the four + un-implemented hot-path methods (`find_matches`, `get_preference`, + `is_satisfied_by`, `get_dependencies`) carry stubs that raise + `NotImplementedError("T4: ...")` etc. — instantiation works, + invocation fails loud with a task-label pointer. + `Candidate` was widened with a new `extras: frozenset[str] = field( + default_factory=frozenset)` field (appended after the existing + fields so frozen-dataclass field-order — no-default-then-default + — holds). The default factory means the existing PEP 691 parser + in `pipenv/resolver/manifest_cache.py:407` and all 31 + `test_candidate.py` tests stay non-breaking (verified: 31/31 + pass). RED→GREEN with `tests/unit/test_pure_python_provider.py` + (8 tests; all pass) covering the three T3 validation bullets plus + the symmetrical cross-shape claim that a `Requirement` and a + `Candidate` with matching `(name, extras)` produce equal + identifiers (the load-bearing `resolvelib` invariant). T13 will + extend this file with per-method coverage of `find_matches`, + `get_preference`, `is_satisfied_by`, and `get_dependencies` once + T4-T7 land. `grep -nE + "^[[:space:]]*(from|import)[[:space:]]+pipenv\.patched\.pip\._internal" + pipenv/resolver/pure_python_provider.py` shows zero matches — + the `AbstractProvider` import goes through + `pipenv.patched.pip._vendor.resolvelib`, which is `_vendor` (a + third-party-vendored dep of patched pip), not `_internal` (pip's + own code). `ruff check` clean on the new files. +- **files edited/created**: + - `pipenv/resolver/pure_python_provider.py` (new — class shell: + keyword-only `__init__` storing the four collaborators + `identify` + returning `(canonical_name, frozenset(extras))`; `find_matches`, + `get_preference`, `is_satisfied_by`, `get_dependencies` are stubs + raising `NotImplementedError("Tn: ...")` for T4-T7 to fill in) + - `pipenv/resolver/candidate.py` (widened: added + `extras: frozenset[str] = field(default_factory=frozenset)` and + a field docstring explaining T3's use; imported `field` from + `dataclasses`) + - `tests/unit/test_pure_python_provider.py` (new — 8 RED→GREEN + tests covering the three T3 validation bullets: identifier shape + for `Requirement` with/without extras + multi-extra, identifier + shape for `Candidate` with default + explicit-empty extras, and + round-trip equality across same/different extras + cross-shape + `Requirement` ↔ `Candidate`; T13 extends) + +--- + +### T4: `PurePythonProvider.find_matches` + +- **depends_on**: `[T1, T3]` +- **location**: `pipenv/resolver/pure_python_provider.py` (extend) +- **description**: + Per design §5.3. Hot path: + 1. Read candidates from `self._cache.get(index_url, name)` (warmed + by `ParallelFetcher` pre-resolve). Cache miss → call + `self._fetcher.populate([(index_url, name)])` for that one + package — single round-trip. + 2. Filter candidates: every `requirement.specifier.contains(version)` + must hold; none of the `incompatibilities` candidates match; + `target_env` marker evaluates True for the candidate's + `requires_python`. + 3. Return iterator ordered high-version-first using + `pipenv.vendor.packaging.version.Version`. +- **validation**: + - Mock cache returns a list of 5 `Candidate`s for `django`; + `find_matches` with `specifier=">=4.0"` returns only those with + version ≥ 4.0, highest first. + - Mock cache returns 0 candidates; `find_matches` returns empty + iterator without raising. + - `incompatibilities` filter: pass one of the returned candidates + as incompatible; it's excluded from the output. +- **status**: Completed +- **log**: + - 2026-05-12 — Implemented per design §5.3. Hot path lives in + :meth:`PurePythonProvider.find_matches` and three private + helpers (`_collect_cached_candidates`, + `_candidate_satisfies_requirements`, + `_candidate_requires_python_ok`). Algorithm walks every + configured ``index_url`` (new ``__init__`` kwarg; see below), + concatenates non-``None`` `cache.get(index_url, name)` hits, + dispatches a single `fetcher.populate([(idx, name) for idx in + index_urls])` on full miss, deduplicates on + ``(name, version, filename)``, filters against the merged + specifier intersection of every `Requirement` in + `requirements[identifier]`, drops candidates whose identity + tuple is in `incompatibilities[identifier]`, applies the + candidate's own `requires_python` SpecifierSet against + `target_env["python_version"]` (with `prereleases=True` so an + alpha CPython target doesn't get silently dropped — mirrors + pip's `evaluate_link`), and sorts by + `pipenv.vendor.packaging.version.Version` descending. Sdist + vs wheel filtering is deliberately NOT done here per Q-A — T7 + raises `_SdistEncountered` when actually expanding deps, and + Q-F's top-level pre-check lives in T9's backend, not on the + provider hot path. + - `__init__` widened with two keyword-only fields: `index_urls: + Sequence[str] = ("https://pypi.org/simple",)` stored as a + tuple (frozen-by-convention), and `allow_prereleases: bool = + False` for the explicit `--pre` opt-in. Defaults preserve + T3's existing call sites — T3's 8 tests construct the provider + without these kwargs and still pass. The test helper + `_make_provider` was extended in lockstep so both T3 and T4 + cases share the same builder. + - Loud-failure stance on `InvalidVersion`: a candidate whose + `version` string can't be parsed by + `packaging.version.Version` is filtered out rather than + sorting last. Rationale: the PEP 691 parser in + `pipenv/resolver/manifest_cache.py` should have rejected it + before reaching the cache; if it slipped through, silently + sorting it last would mask the upstream bug. Same stance + isn't applied to `InvalidSpecifier` on a candidate's + `requires_python` (we accept the candidate on malformed + `requires-python` advertisement) — that mirrors pip's + behaviour where a bad index payload doesn't make the package + invisible. + - Cache-key note for T5/T6 follow-ups: `ParsedManifestCache` is + keyed on `(index_url, name)`; the provider doesn't know which + index served a given cached candidate. T4's walk-every-index + approach is correct but does N cache reads per `find_matches` + call where N = len(index_urls). If this becomes a hot-path + bottleneck under a >5-index workload, the fix is to widen + `Candidate` with an `index_url` field rather than threading + it through here — not a refactor for T5/T6. + - RED→GREEN: 4 new tests landed in + `tests/unit/test_pure_python_provider.py` + (`TestFindMatchesSpecifierFilter`, + `TestFindMatchesEmpty`, + `TestFindMatchesIncompatibilitiesFilter`, + `TestFindMatchesRequiresPythonFilter`) — one per validation + bullet plus the bonus `requires_python` filter. Each uses + a `_FakeCache` + `_FakeFetcher` pair (in-file stand-ins for + `ParsedManifestCache` + `ParallelFetcher`) — no real network, + no real disk. Full file now 12 tests; all pass. `grep -nE + "^[[:space:]]*(from|import)[[:space:]]+pipenv\.patched\.pip\._internal" + pipenv/resolver/pure_python_provider.py` shows zero matches. + `ruff check` clean. +- **files edited/created**: + - `pipenv/resolver/pure_python_provider.py` (extended: + `find_matches` + three private helpers; `__init__` widened + with `index_urls` + `allow_prereleases` keyword-only kwargs; + new imports for `Iterable`, `Mapping`, `Sequence`, + `InvalidSpecifier`, `SpecifierSet`, `InvalidVersion`, + `Version`) + - `tests/unit/test_pure_python_provider.py` (extended: 4 new + tests for T4 + supporting `_FakeCache` / `_FakeFetcher` / + `_cand` helpers; `_make_provider` now forwards the new + `index_urls` + `allow_prereleases` kwargs with defaults so + T3 tests stay unchanged) + +--- + +### T5: `PurePythonProvider.get_preference` + +- **depends_on**: `[T1, T3]` +- **location**: `pipenv/resolver/pure_python_provider.py` (extend) +- **description**: + Per design §5.3 + Q-C (strict mirror). Read pip's + `pipenv/patched/pip/_internal/resolution/resolvelib/provider.py:176 + (get_preference)` end-to-end, mirror the tuple shape. Expected + components in order (lower = preferred): + - `0 if pinned else 1`: pinned-version requirements first. + - `0 if source == "pipfile" else 1`: Pipfile-direct over transitive. + - `count(backtrack_causes for identifier)`: backtrack-causing + identifiers later. + - Lexicographic tie-breaker on identifier name (stable order). +- **validation**: + - Pin vs range: pinned requirement returns a preference tuple that + sorts before a range requirement. + - Pipfile vs transitive: Pipfile-direct sorts before transitive. + - Backtrack: an identifier appearing in `backtrack_causes` 3 times + sorts after one appearing 0 times. +- **status**: Completed +- **log**: + - Read pip's `PipProvider.get_preference` in + `pipenv/patched/pip/_internal/resolution/resolvelib/provider.py` + end-to-end. pip's tuple is a 7-tuple at the time of this commit: + `(not conflict_promoted, not direct, not pinned, not upper_bounded, + requested_order, not unfree, identifier)`. Three of the seven + components depend on pip-internal state we don't carry in Phase 3 + — see parity-divergence candidates below. + - Confirmed `resolvelib.AbstractProvider.get_preference` signature + from `pipenv/patched/pip/_vendor/resolvelib/providers.py:33-90` and + the `RequirementInformation = namedtuple(..., ["requirement", + "parent"])` shape from + `pipenv/patched/pip/_vendor/resolvelib/structs.py:27-40`. + - Tuple shape this commit returns (low = preferred): + `(backtrack_count, not is_pipfile, not is_pinned, not is_upper_bounded, + not is_unfree, identifier_key)` where `identifier_key = (name, + tuple(sorted(extras)))` for deterministic tie-breaking. Mirrors + the four leading axes the plan T5 validation matrix calls out + plus the `upper_bounded` and `unfree` slots that fall out of the + same operator scan (free win — same operator pass). + - RED→GREEN evidence: 6 new tests landed in + `tests/unit/test_pure_python_provider.py` covering pinned-vs-range, + `==4.*` not-pinned, Pipfile-vs-transitive, backtrack-count, + lexicographic tie-break, and the empty-information defensive path. + Initial run with the stub: `NotImplementedError("T5: + PurePythonProvider.get_preference")` on first test. Post-impl: + all 18 tests pass (T3's 8 + T4's 4 + T5's 6). `ruff check` clean. + `grep -nE "^[[:space:]]*(from|import)[[:space:]]+pipenv\.patched\.pip\._internal" + pipenv/resolver/pure_python_provider.py` shows zero matches. + - Parity-divergence candidates (recorded here for T_PARITY_MATRIX + follow-up; pip-side components we deliberately do NOT mirror today): + 1. `conflict_promoted` flag — pip maintains a `_conflict_promoted` + set populated by `narrow_requirement_selection` (provider.py + lines 120-174); we don't ship `narrow_requirement_selection` + in Phase 3 so we render the raw backtrack-cause count instead. + Same ordering intent (frequently-backtracking identifiers + deferred), different storage model. + 2. `direct` flag — pip's `direct` is `isinstance(r, + ExplicitRequirement)` (URL-direct entries). Our typed + `Requirement` doesn't carry URL-direct yet; we render the + closest analog as `source == "pipfile"`. When the dataclass + gains URL-direct (Phase 4 work, not scoped here), the slot + upgrades to a strict mirror. + 3. `requested_order` integer — pip's `_user_requested` map keys + identifiers to user-CLI-input ordering with `math.inf` + fallback. Initiative G doesn't track CLI-order today + (Pipfile is the source of truth, not argv); the slot is + omitted. Likely doesn't affect lockfile byte-identity for + Pipfile-only workflows; T15 / T_PARITY_REAL is the gate. +- **files edited/created**: + - `pipenv/resolver/pure_python_provider.py` (extended: + `get_preference` body replaces the `T5` `NotImplementedError` + stub; method-level docstring cites pip's source file + + function for side-by-side audit) + - `tests/unit/test_pure_python_provider.py` (extended: 6 new + T5 tests + an `_ri()` helper that builds the vendored + `RequirementInformation` namedtuple; module docstring + refreshed to call out T5 scope) + - `initiative-g-phase3-plan.md` (this entry: status → + Completed, log + files filled in, parity-divergence + candidates recorded) + +--- + +### T6: `PurePythonProvider.is_satisfied_by` + +- **depends_on**: `[T1, T3]` +- **location**: `pipenv/resolver/pure_python_provider.py` (extend) +- **description**: + Per design §5.3. Three checks: + - `candidate.version in requirement.specifier`. + - Extras compatibility: every `requirement.extras` must be a subset + of `candidate.provides_extras` (via metadata) OR — for the typical + case where we don't have metadata yet — assume true. Audit pip's + behaviour and mirror. + - `requirement.marker.evaluate(target_env)` is True or `marker is None`. +- **validation**: + - `==4.0.1` requirement, candidate version `4.0.1` → True. + - `==4.0.1` requirement, candidate version `4.0.2` → False. + - `requirement.marker = Marker("python_version < '3.10'")` and + `target_env = {"python_version": "3.12"}` → False. +- **status**: Completed +- **log**: + - 2026-05-12 — Implemented per design §5.3. Replaced the + `NotImplementedError("T6: ...")` stub with the three-check + predicate the plan calls out. Body cites pip's source for the + audit trail: `SpecifierRequirement.is_satisfied_by` at + `pipenv/patched/pip/_internal/resolution/resolvelib/requirements.py:121` + and `PipProvider.is_satisfied_by` at + `pipenv/patched/pip/_internal/resolution/resolvelib/provider.py:300`. + - **Check 1 (version)**: `spec.contains(Version(candidate.version), + prereleases=True)`. Pip-parity rationale: pip passes + `prereleases=True` unconditionally at the predicate because + `PackageFinder` filtered prereleases out upstream. T4's + `find_matches` plays the same `PackageFinder` role for us + (applies the `allow_prereleases` + per-specifier policy), so this + predicate must not re-filter — otherwise a candidate + `find_matches` legitimately handed back would be silently + rejected. `InvalidVersion` returns `False` (same loud-failure + stance as T4's sort step). + - **Check 2 (extras, lazy-metadata stance)**: when + `candidate.extras` is empty we treat it as "metadata not yet + loaded" and admit the candidate; when non-empty we require + `requirement.extras <= candidate.extras`. Phase-3 audit of pip's + `SpecifierRequirement.is_satisfied_by` shows pip itself does NOT + check extras at the predicate — it relies on the `(name, extras)` + identifier grouping in `identify` (T3) to prevent `django` ever + being matched against `django[argon2]`. Our identifier grouping + is identical, so the lazy-metadata "admit" is the conservative + mirror of pip; the strict check on populated `extras` is a free + upgrade for tests / future paths that synthesise extras-bearing + candidates. Recorded as parity-divergence candidate #4 below for + T_PARITY_MATRIX follow-up. + - **Check 3 (marker)**: `marker.evaluate(self._marker_environment())` + where `_marker_environment` overlays `self._target_env` onto + `packaging.markers.default_environment()`. T7 will reuse the + same helper to filter transitive deps. Pip-parity divergence: + pip does NOT evaluate markers at this predicate — marker + filtering happens upstream in `iter_dependencies`. Our typed + `Requirement` doesn't pre-filter (T7 will, but the predicate + must still handle a Pipfile-direct requirement carrying a + marker), so we evaluate defensively. Strictly more conservative + than pip (admits a strict subset of what pip admits) — recorded + as parity-divergence candidate #5 below. + - **Helper extracted**: `_marker_environment(self)` returns the + overlay dict. Lives on the provider so T7 reuses it instead of + re-deriving the overlay logic. Unevaluable markers are caught + in a broad `except Exception:` returning `False` — same defensive + stance pip applies around marker parsing in `req_install`. + - RED→GREEN: 14 new tests landed in + `tests/unit/test_pure_python_provider.py` covering the three plan + bullets plus the extras-lazy-metadata phase-3 decision, prerelease + pip-parity, empty-specifier, marker-true / marker-None / + target-env override paths. Initial RED with the stub: + `NotImplementedError("T6: ...")` on first test. Post-impl: all 32 + tests pass (T3's 8 + T4's 4 + T5's 6 + T6's 14). `ruff check` + clean. `grep -nE + "^[[:space:]]*(from|import)[[:space:]]+pipenv\.patched\.pip\._internal" + pipenv/resolver/pure_python_provider.py` shows zero matches. + - Parity-divergence candidates (additions to the T5 list for + T_PARITY_MATRIX follow-up): + 4. Extras-at-predicate strictness — pip's + `SpecifierRequirement.is_satisfied_by` is specifier-only; we + admit when `candidate.extras` is empty (lazy-metadata mirror) + AND strict-check when populated. Same logical result on the + common path; the strict-check is an extra safety net for + future code that synthesises extras-bearing candidates. + 5. Marker evaluation at predicate — pip filters markers upstream + in `iter_dependencies`; we evaluate at the predicate. Result: + we may reject a candidate pip would have admitted in a + contrived case where a Pipfile-direct requirement carries an + always-false marker. Strictly more conservative; unlikely to + affect lockfile parity in practice (Pipfile-direct markers are + rare). +- **files edited/created**: + - `pipenv/resolver/pure_python_provider.py` (extended: + `is_satisfied_by` body replaces the `T6` stub; new + `_marker_environment` private helper; method-level docstring + cites pip's source files for the side-by-side audit trail; no + new top-level imports — `default_environment` is imported lazily + inside `_marker_environment` to avoid widening the module-import + surface for a function that runs only when a requirement carries + a marker) + - `tests/unit/test_pure_python_provider.py` (extended: 14 new T6 + tests across three test classes — `TestIsSatisfiedByVersion` + (7 tests: exact pin / range / empty spec / prerelease admit + parity), `TestIsSatisfiedByMarker` (4 tests: marker-false / + marker-true / marker-None / target-env override), and + `TestIsSatisfiedByExtras` (4 tests pinning the lazy-metadata + Phase-3 decision: unknown / superset / disjoint / no-extras- + requested); module docstring refreshed to call out T6 scope) + - `initiative-g-phase3-plan.md` (this entry: status → Completed; + log + files filled in; parity-divergence #4 + #5 recorded) + +--- + +### T7: `PurePythonProvider.get_dependencies` + +- **depends_on**: `[T1, T2, T3]` +- **location**: `pipenv/resolver/pure_python_provider.py` (extend) +- **description**: + Per design §5.3. For wheel candidates: call + `self._metadata_fetcher.fetch_metadata(candidate)`, parse + `requires_dist` headers, return iterator of `Requirement` instances + with `source="transitive", parent=candidate.name`. + + For sdist candidates: Q-A's recommendation is fall-back-to-pip. + This task raises a typed `_SdistEncountered` internal exception + carrying the candidate; `PurePythonBackend.resolve` (T9) catches it + and triggers the fallback. + + Filter dependencies by marker evaluation against `target_env` — same + as `is_satisfied_by`. +- **validation**: + - Wheel candidate with `Requires-Dist: numpy>=1.20,<2.0` → returns a + `Requirement(name="numpy", specifier=">=1.20,<2.0", source="transitive")`. + - Wheel candidate with `Requires-Dist: pytest; extra=='dev'` and the + requirement didn't request `dev` extra → marker evaluates False; + dep filtered out. + - sdist-only candidate → raises `_SdistEncountered(candidate)`. +- **status**: Completed +- **log**: + - 2026-05-12: TDD RED→GREEN. Added 9 new tests across 5 classes + (`TestGetDependenciesWheelRequiresDist`, + `TestGetDependenciesMarkerFilter`, + `TestGetDependenciesSdistFailLoud`, + `TestGetDependenciesMalformedRequiresDist`) covering all 4 plan + acceptance bullets + extras-active + multi-Requires-Dist + leaf + (empty Requires-Dist) + non-extra marker + sdist-no-fetcher-call + + malformed Requires-Dist propagates `InvalidRequirement`. + - Implementation: `_SdistEncountered` (`__init__` stores + `.candidate`, message `f"sdist-only candidate {name} {version!r}"`); + `get_dependencies` gates on `candidate.is_wheel` (T1-derived) → + raises `_SdistEncountered` for sdists; for wheels invokes + `self._metadata_fetcher(candidate)` (callable shape — T9 binds + session+cache around T2's `fetch_metadata`); parses each + `Requires-Dist` entry via `packaging.requirements.Requirement`; + marker-filter via `_marker_active_for_extras` mirroring pip's + `pipenv/patched/pip/_internal/metadata/importlib/_dists.py:224` + (no-marker → yield; no parent extras → eval under `{"extra": ""}`; + parent extras → OR-eval per extra); translates parser fields into + T1 `Requirement(source="transitive", parent=)`. + - Zero `pip._internal.*` imports confirmed + (`grep -n "pip\._internal" pipenv/resolver/pure_python_provider.py` + returns empty). Full provider suite: 41 passed (32 prior + 9 new). + Full unit suite: 1405 passed. + - Parity-divergence candidates recorded in the `get_dependencies` + docstring for T_PARITY_MATRIX: (a) pip's `ExtrasCandidate` + self-dependency synthesis (`candidates.py:533`) is implicit in our + `(name, extras)` identifier scheme rather than a separate + requirement; (b) propagation of `InvalidRequirement` matches pip's + `_dists.py:224` behaviour. +- **files edited/created**: + - `pipenv/resolver/pure_python_provider.py` (replaced T7 stub + + added `_SdistEncountered` class + `_marker_active_for_extras` + helper) + - `tests/unit/test_pure_python_provider.py` (+9 tests across 5 + new classes) + +--- + +### T8: `PurePythonProvider` integration smoke + +- **depends_on**: `[T1, T2, T3, T4, T5, T6, T7]` +- **location**: extend `pipenv/resolver/pure_python_provider.py` + with a tiny `_drive_resolver(requirements, provider)` helper, plus + `tests/unit/test_pure_python_provider_smoke.py` (new). +- **description**: + End-to-end smoke: drive `resolvelib.Resolver(provider, ...)` to + resolution against a synthetic in-memory `ParsedManifestCache` + + `MetadataFetcher` mock that returns hand-crafted candidates + + metadata. Assert the resolved graph matches an expected pin set. +- **validation**: + - Resolve `requests`+`certifi`+`urllib3` against a synthetic 3-package + cache where each package has 2-3 versions; expected pins land. + - Resolve a conflict scenario (`a` requires `b<2`; `c` requires + `b>=2`); assert `resolvelib.ResolutionImpossible` raises with + both causes listed. +- **status**: Completed +- **log**: + - Added module-level `_drive_resolver(requirements, provider, *, + reporter=None, max_rounds=100)` helper at the bottom of + `pure_python_provider.py`. Imports `BaseReporter` + `Resolver` + lazily from `pipenv.patched.pip._vendor.resolvelib` (`_vendor`, + not `_internal` — gate clean). Forwards `max_rounds` defaulting + to resolvelib's own 100. Does NOT catch `ResolutionImpossible` + or `_SdistEncountered` — T9's backend owns translation. + - Added `tests/unit/test_pure_python_provider_smoke.py` with two + integration scenarios per the plan T8 validation matrix: + happy-path 3-package resolve (`requests` 2.32.0 + + `certifi` 2024.2.2 + `urllib3` 2.2.0 expected pins) and conflict + (`a→b<2` vs `c→b>=2` → `ResolutionImpossible` with both causes + + parents recorded). + - RED→GREEN evidence: initial run failed with `ImportError: + cannot import name '_drive_resolver'` on both tests (2 failed); + post-implementation run: 2 passed in 0.20s. Existing 41 + `test_pure_python_provider.py` tests stay green. + - `grep -n "pip\._internal" pipenv/resolver/pure_python_provider.py` + → zero matches (`_vendor.resolvelib` only). + - T9 + T13 unblocked (wave 6). +- **files edited/created**: + - `pipenv/resolver/pure_python_provider.py` (added `_drive_resolver` + helper + exported via `__all__`) + - `tests/unit/test_pure_python_provider_smoke.py` (new — 2 tests, + 2 classes) + +--- + +### T9: `PurePythonBackend` + fail-loud sdist handling + Q-F pre-check + +- **depends_on**: `[T8]` +- **location**: `pipenv/resolver/backends/pure_python.py` (new) +- **description**: + Per design §5.4 + Q-A + Q-F decisions. Implements Initiative F's + `Backend` protocol. + + Flow inside `resolve(request)`: + + 1. **Pre-fetch** top-level package candidates + metadata (Q-B) via + `ParallelFetcher.populate`. + 2. **Q-F top-level wheel-availability pre-check**: for every + top-level Pipfile package in `request.packages.specs`, scan its + cached candidates from `ParsedManifestCache`. If ZERO candidates + are wheels matching the target Python + platform, abort: + `ResolverResponse.result = ResolutionError(pip_message=, conflicts=[])`. Catches the common + sdist-only-toplevel case at startup, not 30 s into a doomed + resolve. + 3. Build `Requirement` set from `request.packages.specs`. + 4. Drive `resolvelib.Resolver(PurePythonProvider(...), ...)`. + 5. **Q-A fail-loud sdist handling**: catch `_SdistEncountered` from + `get_dependencies` (raised by T7 when a transitive's only choice + is an sdist). Translate into + `ResolverResponse.result = InternalError(message=, traceback=...)`. No silent + fallback. + 6. **Normal failure mapping**: + - `resolvelib.ResolutionImpossible` → `ResolutionError` with the + conflict list translated. + - Network/transient errors during `get_dependencies` (mid-resolve) + → `InternalError`. + - Other unexpected exceptions → `InternalError` with traceback. + 7. **Success** → `ResolverSuccess(locked=)`. + +- **validation**: + - Mock `PurePythonProvider` to resolve successfully → backend returns + `ResolverResponse(result=ResolverSuccess(locked=...))`. + - Mock provider to raise `ResolutionImpossible` → backend returns + `ResolverResponse(result=ResolutionError(conflicts=...))`. + - Mock provider to raise `_SdistEncountered("foo", "1.2.3")` → + backend returns `ResolverResponse(result=InternalError(message=...))`; + error message contains both the package name and the version; + NO call to pip backend. + - Q-F pre-check: construct a mocked cache where top-level package + `bar` has zero wheel candidates → backend returns + `ResolverResponse(result=ResolutionError(pip_message=...))`; + error message names `bar` AND suggests `pipenv lock --backend pip`. + `resolvelib.Resolver` is NEVER invoked (verified via mock). +- **status**: Completed +- **log**: + - 2026-05-12: Implemented `PurePythonBackend` adapter wrapping the + `PurePythonProvider` (T3-T8) + `MetadataFetcher` (T2) + + `Requirement` (T1) chain. 7-step `resolve(request)` flow per + design §5.4: + 1. Pre-fetch top-level candidates via + `ParallelFetcher.populate([(index, name) for name in + top_level])`. Errors are silently swallowed — `find_matches` + retries on cache miss. + 2. Q-F top-level wheel-availability pre-check: scan cached + candidates across configured indexes for each top-level; abort + with `ResolutionError` if any top-level has candidates but + ZERO are wheels. `_drive_resolver` is never invoked in this + case (verified in `TestQFPreCheck`). + 3. Build `Requirement` set via + `Requirement.from_pipfile_entry(name, spec)`; helper + `_spec_value_to_pipfile_entry` extracts the specifier suffix + from wire-shape pip-install lines (`"requests==2.31.0"` + → `"==2.31.0"`). + 4. Construct `PurePythonProvider` with a `_metadata_fetcher` + closure binding the configured `session` + `MetadataCache` + around `fetch_metadata`. `allow_prereleases` is sourced from + `request.options.pre`. + 5. Drive `_drive_resolver(reqs, provider)`. Three exception + handlers translate to typed responses: + - `_SdistEncountered` → `InternalError` (Q-A fail-loud, **no + pip-backend fallback**); message names the candidate + `==''` + recovery instruction. + - `ResolutionImpossible` (from + `pipenv.patched.pip._vendor.resolvelib`) → + `ResolutionError` with a multi-line `pip_message` formatted + by `_format_resolution_impossible`; one line per cause: + `" - requires "`. `conflicts` + is left empty because `ConflictRecord(package, version, + requires)` doesn't map 1:1 to resolvelib's + `RequirementInformation` shape — `pip_message` carries the + data instead. + - Any other `Exception` → `InternalError` with traceback. + 6. Success path: translate the resolved + `identifier → Candidate` mapping into + `tuple[LockedRequirement, ...]` via `_translate_mapping`: + - `name` ← `candidate.name` + - `version` ← `f"=={candidate.version}"` (mirrors the + pip-backend `_clean_version` auto-prefix at schema.py + lines 213-224) + - `extras` ← `tuple(sorted(identifier[1]))` + - `hashes` ← `tuple(sorted(":"))` from + `candidate.hashes` + - `index` ← first configured index URL (Phase 3 + single-index simplification) + - `markers`/`requires_python`/etc. left at defaults — Phase 3 + conservative shape. + 7. Returns `ResolverResponse(schema_version=SCHEMA_VERSION, + result=ResolverSuccess(kind="success", locked=), + diagnostics=Diagnostics())`. + - 2026-05-12: TDD RED → GREEN. Wrote 4 acceptance tests in + `tests/unit/test_pure_python_backend.py`, verified RED via + `ImportError`, implemented backend, all 4 tests pass. Provider + suite (60 tests) + smoke (2 tests) + Initiative F backend tests + (9 tests) all still pass — no regressions. + - 2026-05-12: Verified `grep -n "pip\._internal" + pipenv/resolver/backends/pure_python.py` returns ZERO matches. + Only patched-pip import is the resolvelib exception type from + `pipenv.patched.pip._vendor.resolvelib` (vendor, permitted). + - 2026-05-12 (T9b follow-up): T10 defaulted the four `__init__` + collaborators to `None` so the registry path + (`get_backend("pure-python")`) could call `cls()` with no args. + That left `resolve()` NPE-ing on `self._fetcher.populate(...)` + when arriving via the registry. Added + `_bootstrap_from_request(request)` (called at the top of + `resolve()`) that self-constructs the four collaborators from the + `ResolverRequest` envelope when they're `None` on `self`: + `PEP691Client(session, verify=...)` + `ParallelFetcher(client, + cache)` + `ParsedManifestCache(PIPENV_CACHE_DIR/pipenv-manifests)` + + `MetadataCache(PIPENV_CACHE_DIR/pipenv-manifests/metadata-v1)`. + Bootstrap is idempotent — pre-injected (test) collaborators win. + Session built via `pipenv.utils.internet.get_requests_session` + (the same factory the production prefetch path uses) with a + `PoolManager` fallback for sandboxed paths. Cache-dir resolution + mirrors `routines/lock.py::_prefetch_index_manifests_if_enabled`. + Added 2 unit tests (`TestBootstrapFromRequest`) — all 28 tests in + `test_pure_python_backend.py` pass; zero `pip._internal` matches + survive. +- **files edited/created**: + - `pipenv/resolver/backends/pure_python.py` (replaced stub with + `PurePythonBackend` class; T9b: added `_bootstrap_from_request` + + `_cache_dir_from_request`) + - `tests/unit/test_pure_python_backend.py` (new — 4 acceptance + tests covering success path, `ResolutionImpossible`, fail-loud + `_SdistEncountered`, and the Q-F top-level wheel pre-check; + T9b: appended `TestBootstrapFromRequest` with 2 rows pinning the + bootstrap path and the pre-injection-wins idempotency contract) + +--- + +### T10: Backend registration + +- **depends_on**: `[T9]` +- **location**: `pipenv/resolver/backends/__init__.py` (extend the + existing `REGISTRY` populate). +- **description**: + Register `PurePythonBackend` under name `"pure-python"`. `pip` stays + the default. Verify the `get_backend("pure-python")` lookup works + and that `--backend pure-python` / `[pipenv] resolver_backend = + "pure-python"` selects it. +- **validation**: + - `python -c "from pipenv.resolver.backends import get_backend; b = get_backend('pure-python'); print(b.name, b.is_available())"` prints `pure-python True`. + - `pipenv lock --backend pure-python` smoke-runs against the 30-pkg + fixture (not parity-checked yet; just no crashes). +- **status**: Completed +- **log**: + - Registered `PurePythonBackend` under name `"pure-python"` in + `pipenv/resolver/backends/__init__.py`. Pip remains the default + via `DEFAULT_BACKEND_NAME = "pip"`. + - Relaxed `PurePythonBackend.__init__` to make all collaborators + (`cache`, `fetcher`, `session`, `metadata_cache`) optional + (defaulting to `None`) so the class is zero-arg-constructible. + The Initiative F registry's `get_backend` helper invokes `cls()` + for class-shaped entries; the dispatcher injects collaborators + before calling `.resolve()` (Phase 4 wiring). All four existing + T9 tests still pass their collaborators via keyword args — fully + backward compatible. + - Added `PurePythonBackend` to the package `__all__`. + - TDD: added `TestBackendRegistration` (2 tests) to + `tests/unit/test_pure_python_backend.py` — RED→GREEN cycle + confirmed. + - Validation smoke: command from acceptance criterion 1 prints + `pure-python True`. `get_backend("pip")` and + `get_backend("pure-python")` both succeed; unknown names still + raise `KeyError` listing both registered backends. + - Regression: full `tests/unit/test_pure_python_backend.py` (6 + tests) and `tests/unit/test_resolver_backends.py` (9 tests) both + pass — 15 / 15. + - Smoke run of `pipenv lock --backend pure-python` is deferred to + T_BENCH / T15 (per orchestrator instruction). +- **files edited/created**: + - `pipenv/resolver/backends/__init__.py` (registered backend, added + to `__all__`). + - `pipenv/resolver/backends/pure_python.py` (relaxed `__init__` + defaults to support zero-arg construction). + - `tests/unit/test_pure_python_backend.py` (appended + `TestBackendRegistration` class — 2 new tests). + +--- + +### T11: Unit tests — `Requirement` + +- **depends_on**: `[T1]` +- **location**: `tests/unit/test_pure_python_requirement.py` (new) +- **description**: + Round-trip, equality, hashability, `from_pipfile_entry` across the + shapes enumerated in T1's spec. Coverage ≥ 95 %. +- **status**: Completed +- **log**: + - Audited T1's 15-test seed against + `pipenv/resolver/pure_python_requirement.py`: starting coverage was + 97.22 % (line 174 — the `TypeError` raise for unsupported value + shapes — was the sole miss). + - Extended the file with 20 additional tests covering the T11 + matrix: unsupported-shape rejection (int / list / None); + dict-form keys the constraint node deliberately ignores + (`editable`, `git`, `ref`, `index`, `path`); empty-string and + missing-`version` dict shapes; whitespace-tolerant specifier + parsing; combined version + extras + markers; `None` and `""` + markers falling through to `marker is None`; name-canonicalisation + corner cases (`Foo.Bar_Baz` → `foo-bar-baz`, `a__b--c..d` → + `a-b-c-d`, `3M` → `3m`); and the `source` Literal escape-hatch + audit (no runtime enforcement, by design — pinned with a + `# T11 audit:` rationale). + - Final coverage on `pure_python_requirement.py`: **100 % line + + 100 % branch (35 tests)**, measured with + `pytest tests/unit/test_pure_python_requirement.py + --cov=pipenv.resolver.pure_python_requirement + --cov-report=term-missing --cov-branch -o addopts=""`. + - No bugs found in T1's implementation while extending coverage — + silent-ignore behaviour for non-constraint dict keys + (`editable`, VCS) matches the docstring contract; documented in + the test file rather than tightened, per T11's brief. +- **files edited/created**: + - `tests/unit/test_pure_python_requirement.py` (extended; + 35 tests total, 100 % coverage). + +--- + +### T12: Unit tests — `MetadataFetcher` + +- **depends_on**: `[T2]` +- **location**: `tests/unit/test_pure_python_metadata.py` (new) +- **description**: + PEP 658 fast path (with mock PEP 658 URL + body); wheel-head + fallback (with synthetic test wheels built in `tmp_path`); cache + round-trip; hash-mismatch behaviour; missing-METADATA inside the + zip fallback path. Coverage ≥ 90 %. +- **status**: Completed +- **log**: + - 2026-05-12: Extended the T2 RED-phase test suite from 9 to 65 + tests, raising line coverage on + `pipenv/resolver/pure_python_metadata.py` from 73 % to 97.75 % + (well above the 90 % gate). New test surface: + `TestParseMetadataText` (full `CoreMetadata` field population, + blank-`Requires-Python` normalisation), `TestCoreMetadataDataclass` + (frozen, slots, hashable + `frozenset` membership, value equality), + `TestPEP658Extra` (empty / `None` / non-sha256 hash dicts, + non-UTF-8 body, HTTP 404, session raises, missing body), + `TestWheelHeadFallbackExtra` (missing METADATA member, corrupt + zip tail, HEAD 405 with no / `*` / garbage Content-Range, HEAD + 405 with probe 4xx, both calls session-failure, HEAD 200 with + no Content-Length, HEAD 200 with malformed Content-Length, + wheel using Windows `\` separator for METADATA), + `TestMetadataCacheExtra` (malformed JSON, non-dict payload, + wrong schema version, missing required key, generic `OSError` + on read, `UnicodeDecodeError`, `cache.put` `OSError` is + non-fatal + debug-logged, `os.replace` failure cleans temp + file, URL → path determinism + uniqueness, fresh-process + round trip), `TestHttpHelpers` (`_get_header` case-insensitive, + raising `get` / `items`, `_http_request` exception → `None`, + `_http_get_range` failure modes), and `TestPartialFile` + (seekable / readable / writable, seek SET / CUR / END + + negative + bad whence, in-buffer read, EOF read, prefix-fetch + growth, prefix oversize trim, prefix short-read raises, + suffix-fetch growth, suffix oversize trim, suffix short-read + raises). Remaining 7 uncovered lines are the + `_MAX_CENTRAL_DIRECTORY_WINDOW`-overrun branch (would need a + >1 MB synthetic wheel), the `total_length <= 0` guard (the + probe path already raises before reaching it), and a couple + of related defensive branches that aren't reachable from the + public API surface. No bugs found in T2's implementation. +- **files edited/created**: + - `tests/unit/test_pure_python_metadata.py` — extended (9 → 65 + tests; +56) + +--- + +### T13: Unit tests — `PurePythonProvider` + +- **depends_on**: `[T8]` +- **location**: `tests/unit/test_pure_python_provider.py` (new) +- **description**: + Per-method coverage of `identify`, `find_matches`, `get_preference`, + `is_satisfied_by`, `get_dependencies`. Use mock cache + mock + metadata fetcher. Edge cases per T3–T7 validation lists. Coverage + ≥ 90 %. +- **status**: Completed +- **log**: + - 2026-05-12: Extended `tests/unit/test_pure_python_provider.py` from + 43 -> 60 tests (17 added) covering the audit list in T13: + duck-typed `Candidate` in `identify`; `find_matches` dedup, + `populate`-less fetcher, `InvalidVersion` skip, `allow_prereleases` + on/off, `target_python=None`, `InvalidSpecifier` in + `requires_python`; `get_preference` `specifier=None`, + `backtrack_causes` skip-when-`requirement-None` / + different-identifier / `identify`-raises; + `is_satisfied_by` unparseable version + marker-raises; + `get_dependencies` blank `Requires-Dist`, marker-raises with + parent-extras + without parent-extras. Coverage on + `pipenv/resolver/pure_python_provider.py`: 88 % -> 99 % + (98.96 % exact; 2 uncovered lines are a deliberately defensive + `InvalidVersion` branch in `_candidate_requires_python_ok` that + cannot be reached against the current vendored `packaging` -- + `SpecifierSet.contains` returns `False` on malformed versions + rather than raising; documented in-file). All 60 unit tests + + 2 smoke tests pass. +- **files edited/created**: + - tests/unit/test_pure_python_provider.py + +--- + +### T14: Unit tests — `PurePythonBackend` + fail-loud sdist + Q-F pre-check + +- **depends_on**: `[T9]` +- **location**: `tests/unit/test_pure_python_backend.py` (new) +- **description**: + - Successful resolve → `ResolverSuccess`. + - `ResolutionImpossible` → `ResolutionError` with conflict list. + - `_SdistEncountered` (transitive) → `InternalError` whose message + contains the package name + version + "sdist-only" + suggested + workaround. NO call to pip backend (verified via mock). + - Q-F top-level pre-check: mocked cache returns zero wheel + candidates for a top-level package → `ResolutionError` with a + message naming the offending package + suggesting + `pipenv lock --backend pip`. `resolvelib.Resolver` never invoked. + - Backend is registered (loadable from `get_backend("pure-python")`). + - Coverage ≥ 90 %. +- **status**: Completed +- **log**: + - T9 seeded the file with 4 acceptance tests (success / conflict / + sdist / Q-F). Initial coverage measured at **88 %** (18 missed + lines). T14 extended the suite with 20 additional focused tests + distributed across 9 new classes — `TestIsAvailable`, + `TestPrefetchExceptionTolerated`, `TestQFPreCheckEdges`, + `TestGenericExceptionTranslatedToInternalError`, + `TestResolutionImpossibleFormattingEdges`, + `TestSpecValueTranslation`, `TestTargetEnvCaching`, + `TestMetadataFetcherClosure`, `TestTranslateMappingEdges` — + each pinning a previously-uncovered branch (Q-B fetcher exception + tolerance, empty-cache Q-F skip, mixed sdist+wheel Q-F skip, + `python_marker_override` formatting, generic exception → + `InternalError`, empty / root-parent / opaque-parent + `ResolutionImpossible` rendering, spec-value translation across + all marker chars + bare-name + empty + unparseable shapes, lazy + vs explicit `target_env`, metadata-fetcher closure forwarding, + non-tuple identifier fallback, version-less candidate skip, + extras + multi-hash propagation, and the empty-`packages.specs` + degenerate request). Final coverage measured at **100 %** (0 + missed lines); all 24 T9+T14 tests pass alongside T10's 2 + `TestBackendRegistration` rows. No bugs found in T9's + implementation; the typed-schema field shapes lined up with the + backend's translations without surprise. +- **files edited/created**: + - tests/unit/test_pure_python_backend.py (extended) + +--- + +### T15: Integration — backend parity on 100-pkg bench + +- **depends_on**: `[T10, T14]` +- **location**: `tests/integration/test_backend_parity.py` (new) +- **description**: + Same Pipfile, lock with both backends, compare lockfiles field-by-field: + - `_meta.hash` byte-equal (both backends consume Pipfile content; + same hash by construction). + - Per-section pins byte-equal (`{name: version}` map). + - Per-package `hashes` set-equal (order-independent). + - Per-package `markers` string-equal (modulo whitespace + canonicalisation). + Divergences logged into `tests/integration/test_backend_parity_known_diffs.md` + with justification. Zero unjustified divergences is the gate. +- **validation**: + - `pytest tests/integration/test_backend_parity.py -v` passes. + - `_known_diffs.md` exists with any justified diffs (empty file is + acceptable). +- **status**: Not Completed +- **log**: +- **files edited/created**: + +--- + +### T_PARITY_REAL: Integration — parity across 10 real projects + +- **depends_on**: `[T15]` +- **location**: + - `tests/integration/test_backend_parity_realworld.py` (new) + - Pipfile fixtures at `tests/integration/fixtures/realworld//Pipfile` +- **description**: + Curated list of 10 real-world top-PyPI projects covering wheel-heavy + / sdist-mix / Django / data-science / async / web frameworks. + Suggested list (to be confirmed in the design-doc PR): + - `django` + `psycopg2-binary` + - `flask` + `gunicorn` + - `fastapi` + `uvicorn` + - `requests` + `httpx` + - `pandas` + `numpy` + - `pytest` + `pytest-cov` + `pytest-django` + - `celery` + `redis` + - `sqlalchemy` + `alembic` + - `cryptography` + `pyOpenSSL` + - `boto3` + `botocore` + Each fixture has a hand-pinned Pipfile. Test runs both backends and + asserts lockfile parity (same gate as T15). +- **validation**: + - All 10 projects' lockfiles match across backends OR the divergence + is justified in `_known_diffs.md`. +- **status**: Not Completed +- **log**: +- **files edited/created**: + +--- + +### T_MATRIX: CI matrix — parity across pip N-1 / N / N+1 + +- **depends_on**: `[T_PARITY_REAL]` +- **location**: + - `.github/workflows/ci.yaml` (extend with a new matrix job) +- **description**: + Add a CI job that runs T15 + T_PARITY_REAL against three pip + versions (the current bundled version + the previous + the next + if pre-release is available). Failure on any matrix entry blocks + Phase 3 sign-off. +- **validation**: + - CI job appears in the workflow. + - Matrix runs to completion on a sample PR. +- **status**: Not Completed +- **log**: +- **files edited/created**: + +--- + +### T_BENCH: Bench measurement under both backends + +- **depends_on**: `[T10]` +- **location**: + - `.github/workflows/ci.yaml` (extend the bench job) + - Commit message records before/after numbers per the design doc §8 gate. +- **description**: + Run `benchmarks/benchmark.py` once under each backend on the same + commit: + - **Run A**: default (pip backend). + - **Run B**: `PIPENV_RESOLVER_BACKEND=pure-python` (or whatever env + surface T10 exposes). + Compare medians of 3 runs each. Phase 3 perf gate (design §8): + `lock-warm` under pure-python ≤ 14.5 s (≥ 30 % off the pre-phase-5 + baseline of 21.3 s). +- **validation**: + - Both bench artifacts exist; the recorded numbers land in the PR + description. + - `lock-warm` median under pure-python meets the gate. +- **status**: Not Completed +- **log**: +- **files edited/created**: + +--- + +### T_PARITY_MATRIX: Parity-matrix doc + +- **depends_on**: `[T15, T_PARITY_REAL]` +- **location**: `docs/dev/initiative-g-phase3-parity-matrix.md` (new) +- **description**: + Enumerates every observed behavioural divergence between the pure- + python and pip backends. Each entry has: + - Brief description. + - Pip-side code reference (file + function). + - Pure-python-side decision (mirror, accept divergence with rationale, + or punt). + - Test reference pinning the choice. +- **validation**: + - Doc shipped; every justified divergence in + `tests/integration/test_backend_parity_known_diffs.md` cross- + references this doc. +- **status**: Not Completed +- **log**: +- **files edited/created**: + +--- + +### T_PLUMBING: User-facing `--backend` + `[pipenv] resolver_backend` dispatcher + +- **depends_on**: `[T10, T14]` +- **location**: + - `pipenv/cli/options.py` (new `--backend` flag on `lock`) + - `pipenv/cli/command.py` (no logic change; ctx already carries + `resolver`) + - `pipenv/routines/lock.py` (read `exec_opts.resolver` -> + `project.settings.resolver_backend` -> `project.settings.resolver`, + pass down) + - `pipenv/utils/resolver.py` (`venv_resolve_deps(resolver_backend=)`, + `_build_resolver_request(resolver_backend=)`, cache-key inclusion) + - `pipenv/utils/settings.py` (`Settings.resolver_backend` accessor) + - `pipenv/resolver/core.py` (child-side + `_resolver_name_from_pipfile` prefers `resolver_backend` over + `resolver`) + - `tests/unit/test_resolver_backends.py` (4 new test classes: + `TestPipfileResolverBackendSetting`, `TestBackendCLIFlag`, + `TestVenvResolveDepsBackendPropagation`, + `TestResolverSubprocessDispatch`) +- **description**: + Wave 7.75 follow-up — adds the user-facing dispatcher T_F.5 left + unwired: + - `pipenv lock --backend [pip|pure-python]` (constrained ``choices=`` + via argparse; typos fail at parse time, NOT at the dispatcher's + KeyError translation), + - `[pipenv] resolver_backend = "pip" | "pure-python"` Pipfile setting + (typed via `Settings.resolver_backend`; coexists with T_F.5's + `[pipenv] resolver` back-compat alias — `resolver_backend` wins + when both are present), + - parent-side resolution of the precedence chain CLI > Pipfile + (`resolver_backend` then `resolver`) > `None` inside `do_lock`, + stamping the result on `ResolverRequest.options.backend` via + `venv_resolve_deps(resolver_backend=...)`. + Default behaviour (no flag, no setting) is byte-identical to + pre-T_PLUMBING: the wire-shape stays empty-string, the cache key + omits the backend component, and `Pipfile.lock` content is identical + byte-for-byte (verified during 2026-05-12 smoke test on a + one-package fixture). + Unblocks T15 / T_PARITY_REAL / T_BENCH to run against the + production CLI surface rather than via in-process `get_backend()` + calls. +- **validation**: + - `pipenv lock --help` shows `--backend {pip,pure-python}`. + - Smoke test on `click = "*"` fixture: `--backend pip` succeeds with + a Pipfile.lock; the no-flag path produces an identical lockfile; + `--backend pure-python` invokes `PurePythonBackend.resolve(...)` + end-to-end (surfaces a backend-side `ResolutionFailure` on `click *` + that is unrelated to plumbing — see Surprises in commit msg). + - 24 unit tests pass under + `tests/unit/test_resolver_backends.py` (15 new + 9 pre-existing). + - `pytest tests/unit/test_pure_python_backend.py` (28 tests), + `pytest tests/unit/test_resolver_core.py` and the rest of the + resolver / settings / lock suites all still green. +- **status**: Completed (2026-05-12) +- **log**: + - 2026-05-12 (Wave 7.75 follow-up): Implemented the CLI flag, + Pipfile setting, and parent-side dispatch chain. All 1439 + relevant unit tests pass. Smoke test on `click = "*"` confirms + default-path byte-identity and end-to-end backend invocation; + pure-python backend itself returns a structured `ResolutionError` + that is a backend-side correctness issue, NOT a plumbing failure + (the typed `pip_message` propagates cleanly to the parent's + `ResolutionFailure` formatter, exactly the Q-F structured-error + shape the design specifies). STOP-and-report per task spec + rather than chasing the backend bug here. +- **files edited/created**: + - `pipenv/cli/options.py` + - `pipenv/routines/lock.py` + - `pipenv/utils/resolver.py` + - `pipenv/utils/settings.py` + - `pipenv/resolver/core.py` + - `tests/unit/test_resolver_backends.py` + - `initiative-g-phase3-plan.md` (this entry) + +--- + +### T_SHIP: Phase 3 ship-bundle + +- **depends_on**: `[T_BENCH, T_PARITY_MATRIX]` +- **location**: + - `news/initiative-g-phase3-pure-python-backend.feature.rst` (new) + - `docs/dev/initiative-g-pure-python-design.md` (umbrella doc) — + flip Phase 3 acceptance bullets to `[x]`. + - `docs/pipfile.md` — entry for `[pipenv] resolver_backend` Pipfile + setting + the `pure-python` option. +- **description**: + Per the T17/T22 convention from Phase 1+2: news fragment, design- + doc status update, user-facing setting documentation. +- **validation**: + - `python -m towncrier --version` succeeds; fragment renders. + - User docs show the new setting with semantics, defaults, and + sdist-fallback note. + - Umbrella design doc shows Phase 3 row checked with measured perf + delta inline. +- **status**: Not Completed +- **log**: +- **files edited/created**: + +--- + +## Parallel Execution Groups + +| Wave | Tasks | Can Start When | +| ---- | ---------------------------------- | ----------------------------- | +| 1 | T1, T2 | Immediately (no deps) | +| 2 | T3, T11, T12 | T1 done (T3, T11), T2 (T12) | +| 3 | T4, T5, T6 | T1 + T3 done | +| 4 | T7 | T1 + T2 + T3 done | +| 5 | T8 | T1–T7 done | +| 6 | T9, T13 | T8 done | +| 7 | T10, T14 | T9 done | +| 8 | T15 | T10 + T14 done | +| 9 | T_PARITY_REAL, T_BENCH | T15 done | +| 10 | T_MATRIX | T_PARITY_REAL done | +| 11 | T_PARITY_MATRIX | T15 + T_PARITY_REAL done | +| 12 | T_SHIP | T_BENCH + T_PARITY_MATRIX done| + +Max concurrency: 3 (Wave 2 — T3 + T11 + T12). Total: 17 tasks. + +--- + +## Testing Strategy + +- **Unit tests** alongside implementation (T11–T14 pin to T1, T2, + T8, T9 respectively). Coverage gate via T17-of-Phase-1's + `.github/workflows/ci.yaml:resolver-module-coverage` job — extend + it to include the new test files + `--cov=pipenv.resolver.pure_python_*` + + `--cov=pipenv.resolver.backends.pure_python`. +- **Integration tests** (T15, T_PARITY_REAL) gated behind real PyPI; + marked `@pytest.mark.needs_internet`. +- **Parity gate** (T15 + T_PARITY_REAL) is the load-bearing Phase 3 + acceptance criterion. Divergences must be justified in writing + (T_PARITY_MATRIX). +- **CI matrix** (T_MATRIX) extends the parity gate across pip + versions. + +--- + +## Risks & Mitigations + +| Risk | Mitigation | +| ---- | ---------- | +| Pip's `get_preference` tie-breaking differs from our mirror in subtle ways → lockfile drift | T15 + T_PARITY_REAL are the byte-identity gates; the parity matrix doc captures every divergence; sign-off requires zero unjustified entries. | +| PEP 658 not advertised on private indexes → wheel-head fallback hit rate is high → slower than expected | T_BENCH measures the actual wheel-head latency; if it's net-harmful on private-index-heavy projects, document and ship as opt-in only. Phase 4 can rework. | +| sdist-fallback masks real regressions (user thinks they're on pure-python but silently fell back) | T9 logs an info-level fallback message; T_SHIP documents the behaviour; `pipenv lock --verbose` will show every fallback so users can audit. | +| `resolvelib` version-bump upstream breaks our `AbstractProvider` impl | We vendor resolvelib via patched-pip; upstream bumps are caught by `tasks/vendoring`. Add a regression test that imports our provider and asserts the abstract methods are still required. | +| `MetadataFetcher` cache corruption / poisoning | Same atomic-write contract as `ParsedManifestCache` (T7 pattern). Wheels are content-addressed (sha256 in the index), so a bad cache entry fails hash check on read. | +| Lockfile parity gate too strict — blocks merge on cosmetic differences (whitespace, key order) | T15 explicitly canonicalises before comparison (sorted-keys JSON, frozenset hashes, normalised marker strings). Anything still differing IS a real divergence. | +| 30 % perf gate not reachable without HTTP/2 | T_BENCH measures honestly; if we land Phase 3 at 20 % instead of 30 %, ship anyway and document. The maintenance-cost reduction stands regardless of the perf delta. | + +--- + +## Out of Scope (this plan only) + +Explicitly **not** part of Phase 3: + +- sdist resolution (falls back to pip backend per Q-A). +- Removing the pip backend (Phase 4 sign-off). +- HTTP/2 transport (separate post-Phase-3 effort). +- Replacing `resolvelib`. +- Cross-platform lockfiles. +- Keyring auth (deferred per Q-D). +- CI bench workflow changes beyond what T_BENCH adds. diff --git a/initiative-g-phase3b-plan.md b/initiative-g-phase3b-plan.md new file mode 100644 index 0000000000..6cfda719a3 --- /dev/null +++ b/initiative-g-phase3b-plan.md @@ -0,0 +1,889 @@ +# Plan: Initiative G — Phase 3b (Sdists + Markers + CI dogfood) + +**Generated**: 2026-05-12 +**Source design doc**: [`docs/dev/initiative-g-phase3-design.md`](docs/dev/initiative-g-phase3-design.md) +**Branch**: `maintenance/code-cleanup-phase6-pure-python-resolver-2026-07` (continues from Phase 3a). +**Scope**: Phase 3b only — fills the gaps Phase 3a punted on so the pipenv test suite can run end-to-end through the pure-python backend. + +--- + +## Overview + +Phase 3a (T1–T14 + T9b + T_PLUMBING, commits `a5ac1eff..8b24acee`) shipped a +working pure-python resolver backend: registered, CLI-dispatched, +end-to-end functional. Smoke test confirms a single-package lock +produces a byte-equal `_meta.hash` vs pip. + +Phase 3a deliberately punted on: + +1. **Sdists**: Q-A "fail loud" was the Phase 3a decision. Phase 3b + inverts it — we now build sdist METADATA on the fly via + `pyproject_hooks` (PEP 517 frontend), so the pure-python backend + handles any package pip would. +2. **Markers**: Phase 3a's `_translate_mapping` emitted no + `markers` field on `LockedRequirement`. Phase 3b emits both + the `Requires-Python` → `python_version >= 'X.Y'` translation + AND propagates the `Requires-Dist` marker that introduced each + transitive. +3. **Lockfile parity finish**: `index` field today writes the URL; + pip writes the source name. Phase 3b looks up source name from + `request.sources`. Hashes today are only the chosen file's + hash; pip emits hashes for all distfiles of the resolved + version. Phase 3b collects them all. + +The Phase 3b acceptance gate is **the existing pipenv test suite +passes under `PIPENV_RESOLVER_BACKEND=pure-python`** — a stronger +signal than T15's 100-pkg parity bench. A non-blocking CI matrix +job surfaces divergences as they appear. + +### Decisions baked into this plan (sign-off 2026-05-12) + +- **Sdist build**: direct PEP 517 via vendored `pyproject_hooks`' + `BuildBackendHookCaller.prepare_metadata_for_build_wheel(...)`. + No new vendoring; no subprocess to `pip wheel`. +- **Marker propagation**: full — `Requires-Python` AND the + `Requires-Dist`-line marker that introduced each transitive. +- **CI matrix**: add the `PIPENV_RESOLVER_BACKEND=pure-python` job + to `.github/workflows/ci.yaml` immediately, `continue-on-error: true`. + Promote to required in a follow-up after it goes green. + +--- + +## Dependency Graph (high-level) + +``` + T_M1 ─┬─ T_M3 ─┐ + │ │ + T_M2 ─┘ ├─ T_M4 ─ T_M5 + │ + T_S1 ─ T_S2 ─ T_S3 ─ T_S4 ─ T_S5 ─ T_S6 + │ + T_CI1 ─ T_CI2 +``` + +Markers track (T_M*) and sdists track (T_S*) are independent and +parallel-safe up to their respective convergence points. CI hookup +waits on both. + +--- + +## Tasks + +### T_M1: Marker propagation — extend Candidate + Requirement + +- **depends_on**: `[]` +- **location**: + - `pipenv/resolver/candidate.py` (extend) + - `pipenv/resolver/pure_python_requirement.py` (extend) +- **description**: + - `Candidate`: confirm `requires_python` is preserved end-to-end + from PEP 691 parse → cache → resolver. It already is (verified + via smoke). No code change needed; pin via test. + - `Requirement`: add an optional `introducing_marker: + Marker | None = None` field. When `Requirement.from_pipfile_entry` + builds a transitive via T_M2, this slot carries the + `Requires-Dist` line's marker (e.g. `extra == 'dev'`, + `sys_platform == 'darwin'`). + - Keep the dataclass frozen; rebuild via `dataclasses.replace` + when the resolver needs a marker-aware variant. +- **validation**: + - `Requirement(name="x", ..., introducing_marker=Marker("python_version<'3.10'"))` + constructs, equals itself, hashes. + - `Requirement.from_pipfile_entry` accepts an optional + `introducing_marker` kwarg. +- **status**: Completed +- **log**: + - Added `introducing_marker: Marker | None = None` to the + `Requirement` frozen dataclass after the existing `parent` + field, preserving `__init__` argument order for all existing + callers (the new field is defaulted, so positional construction + sites continue to work unchanged). + - `from_pipfile_entry` was deliberately NOT widened — the helper + only constructs top-level Pipfile constraints, which never + carry an introducing marker. Transitives are built via the + direct `Requirement(...)` call in + `PurePythonProvider.get_dependencies` (T_M2's job). + - `Marker` is hashable out of the box in `pipenv.vendor.packaging` + (`markers.py:329`) — no `__hash__` override needed; the + frozen-dataclass derived `__hash__` continues to work for + `frozenset[Requirement]` membership with the new field. + - Added a new `TestRequirementIntroducingMarker` test class (9 + tests) to `tests/unit/test_pure_python_requirement.py` + covering: default-to-None on `from_pipfile_entry`, default-to-None + on direct construction without the kwarg, round-trip with the + kwarg, equality / inequality / hash semantics across same / + different / None markers, `frozenset` membership, the + `dataclasses.replace` round-trip pattern T_M2 will use, and the + pin that `from_pipfile_entry` rejects the kwarg. + - Added a new `TestRequiresPythonPreservation` test class (2 + tests) to `tests/unit/test_candidate.py` pinning that + `Candidate.requires_python` survives end-to-end from the PEP 691 + JSON parse (`_parse_pep691_json`) verbatim — both the + populated (`">=3.10"`) and `None` paths. This is the + dependency that T_M3's marker emission relies on. + - Coverage on `pipenv/resolver/pure_python_requirement.py` is + 100 % (37 / 37 statements); the grep gate + `grep -n "pip\._internal" ...` returns 0 matches. + - All 44 tests in `test_pure_python_requirement.py` (35 existing + + 9 new) and all 33 tests in `test_candidate.py` (31 existing + + 2 new) pass. +- **files edited/created**: + - `pipenv/resolver/pure_python_requirement.py` (added + `introducing_marker` field and docstring section) + - `tests/unit/test_pure_python_requirement.py` (added + `TestRequirementIntroducingMarker`) + - `tests/unit/test_candidate.py` (added + `TestRequiresPythonPreservation`) + - `initiative-g-phase3b-plan.md` (this entry) + +--- + +### T_M2: Provider records introducing-marker on transitives + +- **depends_on**: `[T_M1]` +- **location**: `pipenv/resolver/pure_python_provider.py` (extend + `get_dependencies`). +- **description**: + Today `get_dependencies` parses each `Requires-Dist` line and + builds a `Requirement(source="transitive", parent=)`. Phase 3b + also captures the marker from the parsed line — the + `packaging.requirements.Requirement` parser already exposes + `.marker`. Pass that through to the new `introducing_marker` + slot from T_M1. + - The marker-extras-active filter logic stays — it already + short-circuits transitives whose marker evaluates False under + the parent's extras. + - For transitives that DO pass the marker filter, the + `introducing_marker` carries the original marker text so + `_translate_mapping` can emit it on the lockfile entry. +- **validation**: + - Mock `Requires-Dist: pytest; python_version < '3.10'`, + parent extras = `{}`, target_env `python_version='3.9'` → + transitive's `introducing_marker` equals the parsed marker. + - Mock `Requires-Dist: numpy>=1.20` (no marker) → transitive's + `introducing_marker is None`. +- **status**: Completed +- **log**: + - 2026-05-12: Threaded `parsed.marker` into the new + `introducing_marker` slot when `get_dependencies` constructs a + transitive `Requirement`. The legacy `marker` field stays + populated with `parsed.marker` too — the existing T7 contract + (`TestGetDependenciesMarkerFilter::test_extra_marker_kept_when_parent_requested_that_extra`) + pins it; the two fields coexist for Phase 3b so T_M3 can read + `introducing_marker` for lockfile emission without disturbing + T6's `is_satisfied_by` evaluator path. Added a dual-marker + comment block at the construction site documenting the split. + RED→GREEN via 4 new tests in + `TestGetDependenciesIntroducingMarker`; 62/62 unit tests pass; + module coverage 97% (floor 90%); zero `pip._internal` imports. +- **files edited/created**: + - `pipenv/resolver/pure_python_provider.py` (extended + `get_dependencies` Requirement construction with + `introducing_marker=parsed.marker` + dual-marker comment block). + - `tests/unit/test_pure_python_provider.py` (added + `TestGetDependenciesIntroducingMarker` with 4 tests covering + marker present, marker absent, extras-filter unchanged, and + extras-context active marker). + +--- + +### T_M3: `_translate_mapping` emits markers + +- **depends_on**: `[T_M2]` +- **location**: `pipenv/resolver/backends/pure_python.py` (extend + `_translate_mapping`). +- **description**: + For each resolved `(identifier, candidate)`: + - Read `candidate.requires_python` (a string like `>=3.10`). + Convert to a marker string: `>=3.10` → `python_version >= '3.10'`, + `>=3.8,<4` → `python_version >= '3.8' and python_version < '4'`, + etc. Use `pipenv.vendor.packaging.specifiers.SpecifierSet` + iteration + a small `_op_to_marker` mapping. + - Look up the `Requirement` instance(s) that selected this + candidate — `resolvelib.Result.criteria` (or whatever the + vendored resolvelib exposes) holds the requirements; combine + each non-None `introducing_marker` via `and`. For multiple + introducing markers (a candidate satisfies multiple + requirements), `OR` them with parentheses. + - Combine Requires-Python marker AND introducing marker(s) into + a single canonical marker string. + - Emit as `markers="..."` on the `LockedRequirement`. +- **validation**: + - Candidate with `requires_python=">=3.10"`, no introducing + marker → `markers == "python_version >= '3.10'"`. + - Candidate with `requires_python=">=3.8"`, one introducing + marker `python_version < '3.12'` → `markers == + "python_version >= '3.8' and python_version < '3.12'"` (or + a canonically-equivalent form). + - Candidate with no `requires_python`, no introducing marker → + `markers is None`. +- **status**: Completed +- **log**: + - 2026-05-12: TDD RED→GREEN — added `TestTranslateMappingMarkers` + (7 tests) covering the full validation matrix: Requires-Python + alone, Requires-Python range (multi-spec `>=3.8,<4`), introducing + marker alone, both sources combined (AND), multiple introducing + markers (OR-joined with parentheses), neither source contributes + (`markers is None`), unparseable `requires_python` falls back to + `None`. Three tests were RED at the start; two passed vacuously + because the pre-T_M3 backend already emitted `markers=None` and + the test asserted that exact shape — those remain in the suite as + pinned regression markers (T_M4 cannot accidentally start emitting + a marker on a no-source candidate without tripping them). + - **Signature change**: `_translate_mapping` now accepts the full + `resolvelib.Result` namedtuple (was `mapping: dict`). Backward- + compat shim — if the caller hands us a bare dict, we treat it as + the mapping with empty criteria (used by `_FakeResult(mapping=...)` + fixtures still hanging around in `TestTranslateMappingEdges`). + - Three new module-level helpers carry the marker-translation logic + out of the per-candidate loop body: + - `_requires_python_to_marker(requires_python)` — `SpecifierSet` + iteration with sorted, stable output and defensive + `InvalidSpecifier`/`ValueError` fallback to `None`. + - `_introducing_marker_for(criterion_information)` — walks + `Criterion.information` rows, pulls non-`None` + `introducing_marker` strings, dedupes preserving insertion + order; single marker → verbatim, multiple → parenthesised + `or`-join (rationale documented inline: union of preconditions). + - `_combine_markers([reqpy, intro])` — `and`-join non-`None` + contributions; single contribution returned verbatim. + - **Multi-introducing-marker join rule**: `or` (parenthesised per + clause). A candidate is admitted to the lockfile because every + introducing requirement was active during resolution, but at + install time the candidate is needed if any of them is active — + hence the union (logical OR) of preconditions. Parentheses + preserve precedence when downstream consumers AND-merge this with + other marker clauses (e.g. the Requires-Python contribution + itself, or T_M4's future per-source markers). + - **Quoting note**: Requires-Python contributions use Python `repr` + (single quotes) while introducing markers come through + `packaging.markers.Marker.__str__` (double quotes). Both are PEP + 508 valid; we preserve each source's canonical form rather than + normalising — keeps the translator allocation-free on the hot + path. Tests pin the exact output strings for stability. + - End-to-end smoke (`/tmp/t-puret-smoke`, `click=*` Pipfile): lock + emits `"markers": "python_version >= '3.10'"` matching pip's + `python_version >= '3.10'` output. Was `null` pre-T_M3. + - Coverage on `pipenv/resolver/backends/pure_python.py` rose from + ~93% (T9 baseline) to 96.43% with the new T_M3 helpers + tests + folded in. The 8 remaining uncovered lines are bootstrap-only + branches (PEP691Client construction, session-fallback paths) that + only fire in the registry production path — not T_M3's surface. + - `grep -n "pip\._internal" pipenv/resolver/backends/pure_python.py` + → 0 matches. +- **files edited/created**: + - `pipenv/resolver/backends/pure_python.py` (extended + `_translate_mapping` signature + body; added 3 module-level marker + helpers; updated `resolve()` call site). + - `tests/unit/test_pure_python_backend.py` (extended `_FakeResult` + with optional `criteria` kwarg + new `_FakeCriterion` helper; + added `TestTranslateMappingMarkers` class with 7 tests). + +--- + +### T_M4: Index URL → source name lookup + +- **depends_on**: `[T_M3]` +- **location**: `pipenv/resolver/backends/pure_python.py` (extend + `_translate_mapping`). +- **description**: + Build a `dict[str, str]` map of `url → name` from `request.sources` + at the top of `resolve()`. Pass to `_translate_mapping`. For each + resolved candidate, emit `index=` instead of the + raw URL. Default to `default_index` if the URL doesn't match + any source (defensive). +- **validation**: + - `request.sources = [Source(name="pypi", url="https://pypi.org/simple")]` + + resolved candidate from `https://pypi.org/simple` → + `LockedRequirement.index == "pypi"`. + - URL not in `request.sources` → emit the URL as-is (defensive + fallback documented). +- **status**: Completed +- **log**: + - 2026-05-12: Built `url_to_name = {s.url: s.name for s in + request.sources}` at the top of `resolve()`; widened + `_translate_mapping(self, result, index_urls, url_to_name=None)` + (kept `url_to_name` keyword-defaulted so legacy fixtures that call + the helper directly still work). Inside the candidate loop the + translator now emits `index = url_to_name.get(default_index, + default_index)` — source NAME when mapped, URL verbatim as the + defensive fallback when no source matches (Phase 3 Candidate + doesn't track per-candidate provenance; Phase 4 follow-up will). + - TDD RED→GREEN: added `TestTranslateMappingIndexName` (3 cases: + URL→name mapping, unmatched-URL fallback, empty-sources fallback). + Updated `TestTranslateMappingEdges::test_candidate_with_extras_and_multiple_hashes` + which had pinned the old URL-as-index behaviour (now pins `"pypi"`). + - Smoke (mandatory): `cd /tmp/t-puret-smoke && rm -f Pipfile.lock && + PIPENV_VERBOSITY=-1 timeout 60 python -m pipenv lock --backend + pure-python` ⇒ `index: pypi`, `markers: python_version >= '3.10'` + (pip-parity confirmed). + - 38/38 tests pass in `tests/unit/test_pure_python_backend.py`; + coverage on `pipenv/resolver/backends/pure_python.py` = 96 % + (well above the 90 % gate). + - `grep -n "pip\._internal" pipenv/resolver/backends/pure_python.py` + = 0 matches (no patched-pip internals coupling introduced). +- **files edited/created**: + - `pipenv/resolver/backends/pure_python.py` (modified) + - `tests/unit/test_pure_python_backend.py` (modified) + +--- + +### T_M5: Marker translation unit tests + +- **depends_on**: `[T_M4]` +- **location**: `tests/unit/test_pure_python_backend.py` (extend) + + `tests/unit/test_pure_python_provider.py` (extend). +- **description**: + - Backend tests for `_translate_mapping` marker emission + (Requires-Python alone; introducing marker alone; both + combined; neither → `markers is None`). + - Provider tests for `get_dependencies` setting + `introducing_marker` correctly across the four shapes + (no marker, marker-only, marker+extras, marker+target-env). + - Index-name lookup tests covering URL→name + the defensive + fallback. +- **validation**: coverage stays ≥ 90 % on the two modules. +- **status**: Completed +- **log**: + - 2026-05-12: Coverage audit pre-T_M5 was already 96 % on + `pure_python.py` (218 stmts, 8 miss) and 97 % on + `pure_python_provider.py` (190 stmts, 6 miss) — well above the + 90 % gate. T_M5 closed the formal scope by pinning belt-and- + braces edge cases that cross the T_M3 / T_M4 / T_S5 boundaries + rather than chasing additional coverage on already-tested paths. + - Added 5 backend edge-case tests + (`TestTranslateMappingT_M5Edges`): + * **Full combo** — `requires_python=">=3.10"` + an introducing + marker (`sys_platform == 'darwin'`) + wheel+sdist sibling + hashes + URL→name source mapping all compose in a single + `LockedRequirement`. Smoke gate over the entire translator. + * **`!=` operator** — `requires_python="!=3.11"` → + `python_version != '3.11'`. Pins the rare-but-real + not-equal operator path. + * **`~=` compatible-release** — `requires_python="~=3.10"` → + `python_version ~= '3.10'`. Pins the implementation's + verbatim rendering (a future canonicaliser that expanded + `~=` to a lower+upper-bound pair would break this gate + deliberately). + * **Whitespace tolerance** — `">= 3.10"` (stray whitespace + around the operator) → canonical + `python_version >= '3.10'` (`SpecifierSet` normalises on + parse). + * **No cross-pollination** — two resolved candidates sharing + the same `requires_python` value get independent markers; + a per-candidate introducing marker on one does NOT leak + into the other's lockfile entry. + - Added 2 provider edge-case tests + (`TestGetDependenciesT_M5IntroducingMarkerCompound`): + * **Compound python+platform marker preserved intact** — + `Requires-Dist: pytest; python_version >= '3.10' and + sys_platform == 'darwin'` with parent extras=∅ ⇒ ONE + `Requirement` whose `introducing_marker` is the FULL + compound `Marker` (both clauses round-trip; not split). + * **Compound extra+platform marker, parent extras active** — + `Requires-Dist: pytest; extra == 'dev' and python_version + >= '3.10'` with parent `extras=frozenset({'dev'})` ⇒ + requirement YIELDED AND `introducing_marker` carries the + FULL compound marker including the `extra==` clause that + the extras filter just consumed (intentional — pip + install re-evaluates at install-time). + - **Post-T_M5 coverage**: 96 % on `pure_python.py` (218 stmts, + 8 miss; missing lines are PoolManager fallback, PIPENV_CACHE_DIR + env-var branch, and three defensive branches inside + `_introducing_marker_for` — all outside the T_M3/T_M4/T_S5 + scope) and 97 % on `pure_python_provider.py` (190 stmts, 6 + miss; missing lines are `_drive_resolver`'s no-reporter + default and an `InvalidVersion` branch in + `_candidate_requires_python_ok` — both outside T_M5 scope). + 115 tests pass (108 pre-T_M5 + 7 new). No production-code + changes; no real bugs surfaced. +- **files edited/created**: + - `tests/unit/test_pure_python_backend.py` (modified) + - `tests/unit/test_pure_python_provider.py` (modified) + +--- + +### T_S1: `pure_python_sdist.py` — sdist METADATA extractor + +- **depends_on**: `[]` +- **location**: `pipenv/resolver/pure_python_sdist.py` (new). +- **description**: + Direct PEP 517 build via vendored `pyproject_hooks`. + - Public surface: + `extract_metadata_from_sdist(candidate, session, *, cache=None) -> CoreMetadata`. + - Flow: + 1. Cache hit → return. + 2. Download the sdist body to a tempdir via `session.get(url)` + (re-use the polymorphism helpers from `pure_python_metadata`). + 3. `tarfile.open` / `zipfile.ZipFile` extract into another + tempdir. + 4. Locate `pyproject.toml`; parse the `[build-system]` table. + If missing, use the legacy `setuptools.build_meta:__legacy__` + fallback (PEP 517 §10). + 5. Install build requirements into an isolated venv? **NO** — + Phase 3b skips build isolation. The user's environment must + carry build-time deps (or the package's `pyproject.toml` + names them and we trust pip's installation history). This + is a known tradeoff vs pip; document in T_PARITY_MATRIX. + 6. Run `BuildBackendHookCaller(srcdir, backend_name).prepare_metadata_for_build_wheel(out_dir)`. + 7. Read `METADATA` from `//METADATA`. + 8. Parse via `_parse_metadata_text` (shared with the wheel + path). + 9. `cache.put` if cache is provided. + - All temp dirs cleaned up in a `finally`. + - Errors raise `SdistBuildError(MetadataFetchError)` with the + backend's stderr if available, so failures are actionable. +- **validation**: + - Against a real PyPI sdist (e.g. a small pure-Python package + that publishes only sdists), `extract_metadata_from_sdist` returns + a `CoreMetadata` with `requires_dist` populated. + - Synthetic sdist fixture in `tmp_path` (build via + `python -m build --sdist` in conftest, or hand-craft a minimal + tarball with a `pyproject.toml` and a `setuptools` build-system + pointer): metadata extracts cleanly. + - Build failure (sdist with a syntax error or unbuildable + `pyproject.toml`) → `SdistBuildError` with the backend's stderr + visible. +- **status**: Completed +- **log**: + - 2026-05-12: TDD RED→GREEN — wrote 27 tests covering the 7-case + matrix from the plan brief (happy path, cache round-trip, HTTP + failure, corrupt archive, build-backend failure, no-pyproject + legacy fallback, path traversal) plus 9 defensive-branch tests + (zip happy path, empty body, non-string backend, backend-path + handling, empty zip member, missing METADATA, non-UTF-8 METADATA, + cache write OSError, filename-from-URL edge cases, timeout via + monkeypatched hook caller). + - PEP 517 frontend: vendored + `pipenv.patched.pip._vendor.pyproject_hooks.BuildBackendHookCaller` + (constructor: `source_dir, build_backend, backend_path=None, + runner=None, python_executable=None`). + `prepare_metadata_for_build_wheel(metadata_directory)` returns the + relative dist-info subfolder name as a `str`. + - **No build isolation** in Phase 3b: the vendored + `BuildBackendHookCaller` has no isolation knob; its `runner` + callable simply subprocess-spawns the in-process hook script using + `sys.executable` in the current Python env. Build-time deps must + be carried by the resolver user's environment. Documented in the + module docstring. + - Timeout: 300 s via `concurrent.futures.ThreadPoolExecutor`; surfaces + as `SdistBuildError("sdist build timed out after 300s ...")`. + - Path-traversal protection: manual pre-extract validation of every + member name — reject empties, absolute paths, and any `..` + segment — applied uniformly to both tar (`tarfile`) and zip + (`zipfile`) archives. Tar extraction also uses Python 3.12+'s + `filter="data"` for defense in depth. + - Source-root enforcement: sdist convention requires exactly one + top-level directory; multi-dir / file-at-root tarballs reject. + - Coverage: 94 % on the new module (target was ≥ 90 %). Remaining + misses are the py3.10 tomli fallback, the older-Python tar filter + `TypeError` rescue, and a few tarfile-only defensive branches + (symlink/dev/fifo) that are awkward to portably construct. + - Adjacent suites unaffected: `tests/unit/test_pure_python_metadata.py` + 65/65 still green. +- **files edited/created**: + - `pipenv/resolver/pure_python_sdist.py` (new, 150 statements) + - `tests/unit/test_pure_python_sdist.py` (new, 27 tests) + +--- + +### T_S2: Integrate sdist path into `MetadataFetcher.fetch_metadata` + +- **depends_on**: `[T_S1]` +- **location**: `pipenv/resolver/pure_python_metadata.py` (extend + `fetch_metadata`). +- **description**: + Today `fetch_metadata` always treats candidates as wheels. Phase 3b: + - Branch on `candidate.is_wheel`. + - Wheel: existing PEP 658 + wheel-head fallback. + - Sdist: delegate to `extract_metadata_from_sdist(candidate, session, + cache=cache)` (T_S1). + - Cache is shared — same on-disk `MetadataCache` keyed by URL + sha256. +- **validation**: + - Wheel candidate routes through existing path (regression test). + - Sdist candidate routes through T_S1; result is a + `CoreMetadata` indistinguishable in shape from a wheel result. + - Cache hit on a sdist doesn't re-build. +- **status**: Completed +- **log**: + - Branch sits after the cache short-circuit and before the wheel + PEP 658 / wheel-head paths. Wheel path is byte-identical to + pre-T_S2; sdist path delegates to + `pure_python_sdist.extract_metadata_from_sdist(candidate, + session, cache=cache)` via a local import (so `pyproject_hooks` + + `tarfile` + `zipfile` aren't paid for on wheel-only resolves). + - Cache lookup happens in `fetch_metadata` first, so a populated + entry short-circuits both the heavy import AND the extractor + entirely. T_S1 also calls `cache.get` internally — that's the + intentional "double-dip" the plan calls out; the second call + only fires on a cold-cache sdist, where the cost is dwarfed by + the download + build step that follows. + - 3 new tests in `TestFetchMetadataSdistRouting`: + `test_sdist_candidate_routes_to_sdist_extractor` (no HTTP + issued on sdist URL; result propagated verbatim), + `test_wheel_candidate_does_not_route_to_sdist` (extractor + patched to raise — never invoked on wheel candidate), + `test_sdist_cache_passed_through` (cache kwarg forwarded; + pre-populated cache short-circuits extractor entirely). + - Coverage on `pure_python_metadata.py` is 96 % (≥ 90 % gate). All + 65 pre-existing tests still pass; T_S1's 27 sdist tests still + pass (untouched). + - `grep "pip\._internal" pipenv/resolver/pure_python_metadata.py` → + 0 import matches (one docstring mention of the constraint + survives, by design). +- **files edited/created**: + - `pipenv/resolver/pure_python_metadata.py` (added sdist branch + + docstring section in `fetch_metadata`) + - `tests/unit/test_pure_python_metadata.py` (added + `_make_sdist_candidate` helper + `TestFetchMetadataSdistRouting` + class with 3 tests) + +--- + +### T_S3: Remove `_SdistEncountered` fail-loud from provider + +- **depends_on**: `[T_S2]` +- **location**: `pipenv/resolver/pure_python_provider.py` (modify + `get_dependencies`) + `pipenv/resolver/backends/pure_python.py` + (remove the Q-A handler). +- **description**: + - Provider: `get_dependencies` no longer raises + `_SdistEncountered` on sdist candidates. It simply calls + `self._metadata_fetcher(candidate)`, which routes through T_S2 + transparently. + - Keep `_SdistEncountered` exported as a typed exception for + backwards-compat (some tests reference it), but mark it + deprecated and never raise it from production code. + - Backend: remove the `except _SdistEncountered` block from + `resolve()`. Update the docstring. +- **validation**: + - Sdist candidate → `get_dependencies` returns the transitive + `Requirement`s like a wheel candidate. + - The 9 T7 unit tests stay green; the one that expected + `_SdistEncountered` is converted to assert the new behaviour + (no exception, transitives returned via T_S2). +- **status**: Completed +- **log**: + - Provider: removed the `if not candidate.is_wheel: raise + _SdistEncountered(candidate)` guard from + `PurePythonProvider.get_dependencies`; sdist candidates now flow + through `self._metadata_fetcher` symmetrically with wheels. The + `_SdistEncountered` class is preserved with a deprecation docstring + pointing at T_S1+T_S2. `_drive_resolver` docstring updated to drop + the `_SdistEncountered` Raises clause. + - Backend: deleted the `except _SdistEncountered` block in + `PurePythonBackend.resolve` and removed `_SdistEncountered` from + the import list; the module-level docstring flow narrative now + points step 4 at T_S1+T_S2's transparent sdist build path. + - Tests rewritten as positive tests (no exception, transitive + `Requirement` set returned via the metadata fetcher); added a + back-compat importability pin for the deprecated class. + - Validation: 100/100 unit tests across + `test_pure_python_provider.py`, `test_pure_python_backend.py`, + and `test_pure_python_provider_smoke.py` pass. Smoke + `pipenv lock --backend pure-python` against a `click="*"` + Pipfile still completes successfully. Zero `pip._internal` + imports remain in either file. +- **files edited/created**: + - `pipenv/resolver/pure_python_provider.py` (removed raise site, + deprecated `_SdistEncountered` class docstring, updated + `get_dependencies` + `_drive_resolver` docstrings) + - `pipenv/resolver/backends/pure_python.py` (removed + `_SdistEncountered` import + `except` handler, updated module + docstring flow narrative) + - `tests/unit/test_pure_python_provider.py` + (`TestGetDependenciesSdistFailLoud` → + `TestGetDependenciesSdistRoutesThroughMetadataFetcher`; converted + `test_sdist_candidate_raises_sdist_encountered` → + `test_sdist_candidate_routes_through_metadata_fetcher`, + `test_sdist_check_does_not_invoke_metadata_fetcher` → + `test_sdist_invokes_metadata_fetcher_exactly_once`; added + `test_sdist_encountered_still_importable` back-compat pin) + - `tests/unit/test_pure_python_backend.py` + (`TestSdistEncounteredFailLoud` → + `TestSdistTransitiveResolvesThroughMetadataFetcher`; converted + `test_sdist_encountered_translates_to_internal_error` → + `test_sdist_transitive_resolves_through_metadata_fetcher`) + - `initiative-g-phase3b-plan.md` (this entry) + +--- + +### T_S4: Remove Q-F top-level wheel pre-check (or repurpose) + +- **depends_on**: `[T_S3]` +- **location**: `pipenv/resolver/backends/pure_python.py` (modify + `resolve` step 2). +- **description**: + Now that sdists work, the Q-F top-level wheel-availability + pre-check is no longer a failure gate. Two options: + - **A**: delete entirely. + - **B**: repurpose for "no candidates at all for a top-level + package" (e.g., typo / yanked-only / network blackout). + Pick B — better UX. Rename internally to + `_top_level_emptiness_pre_check` and update the error message + to say "no candidates found" rather than "no wheel available". +- **validation**: + - Top-level package with only sdists → resolves (no pre-check + fire). T14's Q-F test is converted: pass a top-level package + with ZERO candidates → still fails loud with the new message. +- **status**: Completed +- **log**: + - 2026-05-12 — Repurposed step 2 of `PurePythonBackend.resolve` + from "Q-F top-level wheel-availability pre-check" to + "top-level emptiness pre-check". Renamed the loop's working + list `sdist_only_top` → `empty_top_level` and replaced the + inner wheel-detection (`is_wheel`) with a presence check via + `any(True for _ in manifest.candidates)`. The new + `pip_message` reads "no candidates found in the configured + index[es]" with singular/plural toggling on + `len(request_index_urls)` and recommends + `pipenv lock --backend pip`. Phase 3a's now-dead + `_describe_target_python` / `_describe_target_platform` + helpers were removed (their only callers were the Q-F error + formatter). Tests: rewrote `TestQFPreCheck` to pin two rows + (sdist-only top-level resolves; zero-candidate top-level + fails); rewrote `TestQFPreCheckEdges` to pin four rows + (mixed sdist+wheel resolves; sdist-only across all indexes + resolves; multi-top-level with one empty fires naming only + the empty offender; multi-index plural-spelling). No + `pip._internal` import (grep clean). Click smoke + (`test_url_maps_to_source_name`) still passes — pre-check + correctly does not preempt wheel-bearing top-levels. +- **files edited/created**: + - `pipenv/resolver/backends/pure_python.py` (resolve step 2 + refactor, module docstring updated, dead `_describe_target_*` + helpers removed) + - `tests/unit/test_pure_python_backend.py` (`TestQFPreCheck` and + `TestQFPreCheckEdges` rewritten; module docstring updated) + - `initiative-g-phase3b-plan.md` (this entry) + +--- + +### T_S5: `_translate_mapping` collects all-distfile hashes + +- **depends_on**: `[T_S4]` +- **location**: `pipenv/resolver/backends/pure_python.py` (extend + `_translate_mapping`). +- **description**: + Pip's lockfile convention: emit hashes for every distfile of the + resolved version (wheel + sdist + any other wheel variants). + Phase 3a emitted only the resolved candidate's single hash. + Phase 3b: + - For each resolved `(identifier, candidate)`, look up ALL + candidates from the cache matching `(candidate.name, + candidate.version)` across configured indexes. + - Collect every `(algo, value)` hash from those candidates' + `hashes` frozensets. + - Emit as `tuple(sorted(":"))` on the + `LockedRequirement`. + - Skip duplicates (the same wheel file may appear in multiple + indexes' caches with the same hash). +- **validation**: + - Candidate `click==8.3.3` cached with a wheel + an sdist; both + appear in `LockedRequirement.hashes`. + - Same candidate without a sibling sdist in the cache → only + the wheel hash. Document this as the "we hash what we + actually saw" semantic (vs pip's "we hash whatever the index + advertises", which is the same thing in practice — the cache + IS the index slice we've read). +- **status**: Completed +- **log**: + - 2026-05-12 — Replaced the single-candidate hash-collection block + in `_translate_mapping` with an all-distfile scan that walks + every configured `index_url` and unions `hashes` frozensets from + cache siblings whose `version` matches the resolved candidate's. + Defensive fallback: if the sibling scan turns up nothing (cold + cache / test-fixture-injected workflow) we emit the resolved + candidate's own hashes, preserving Phase 3a fixture flows. + Added `TestTranslateMappingAllDistfileHashes` (5 cases) covering + wheel+sdist siblings, version-scoped matching, multi-index + dedup, cold-cache fallback, and the empty-hashes edge. End-to-end + smoke on `click=*` now emits both wheel and sdist hashes + (`sha256:398329ad…` + `sha256:a2bf429b…`) — byte-equal to pip's + lockfile. The legacy + `TestTranslateMappingEdges.test_candidate_with_extras_and_multiple_hashes` + cache fixture moved from a stub `_wheel_candidate` (single + `0*64` hash) to the test's `candidate_with_extras` (two hashes, + sha256 a/b) so the semantic shift doesn't drop a real test + assertion. Coverage on `pure_python.py`: 96 % (≥ 90 % gate). +- **files edited/created**: + - `pipenv/resolver/backends/pure_python.py` + - `tests/unit/test_pure_python_backend.py` + - `initiative-g-phase3b-plan.md` + +--- + +### T_S6: Sdist + hashes integration tests + +- **depends_on**: `[T_S5]` +- **location**: + - `tests/unit/test_pure_python_sdist.py` (new) + - `tests/unit/test_pure_python_backend.py` (extend) +- **description**: + - Unit tests for `pure_python_sdist`: synthetic sdist build, + cache round-trip, build-failure error path. + - Backend tests: T_S5 hash-collection edge cases (single hash, + multiple hashes, cross-index dedup). + - Coverage gate stays ≥ 90 % on both modules. +- **validation**: tests pass; coverage gate maintained. +- **status**: Completed +- **log**: + - Audited existing coverage post-T_S5/T_M5: sdist 94 %, backend 96 %. + Both already above the 90 % floor; explicit T_S6 deliverables + (synthetic sdist build, cache round-trip, build-failure path, + hash-collection edges) were already present via earlier waves' + test files. + - Added four defensive gap-fill tests to + `tests/unit/test_pure_python_sdist.py::TestDefensiveBranches` + pinning the remaining sdist branches: archive write OSError, + tar device-type member rejection, tar symlink traversal, + and `_locate_source_root` empty-extraction. + - Added three bootstrap-edge tests to + `tests/unit/test_pure_python_backend.py::TestBootstrapFromRequest` + for the PoolManager fallback and the `PIPENV_CACHE_DIR` + env-var path. + - Post-gap-fill coverage: sdist 94 → 97 %, backend 96 → 98 %. + Remaining missing lines are the py3.10 `tomli` / older + `tarfile.extractall` fallbacks plus three marker-translation + guards — none of them are sdist or hash-collection code paths. + - Full resolver unit suite: 261 passed, zero regressions. +- **files edited/created**: + - `tests/unit/test_pure_python_sdist.py` (extended) + - `tests/unit/test_pure_python_backend.py` (extended) + +--- + +### T_CI1: CI matrix entry — `PIPENV_RESOLVER_BACKEND=pure-python` + +- **depends_on**: `[T_M5, T_S6]` +- **location**: `.github/workflows/ci.yaml` +- **description**: + Add a new matrix entry alongside the existing test job that runs + the full pipenv test suite with the env var set: + ```yaml + - name: tests (pure-python backend) + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' + continue-on-error: true + env: + PIPENV_RESOLVER_BACKEND: pure-python + run: ... ... + ``` + Confirm `PIPENV_RESOLVER_BACKEND` env var is honoured by the + T_PLUMBING dispatcher; if not, add the env-var path (env > + CLI > Pipfile > default). + Start with `python-version == '3.12'` only to keep CI minutes + reasonable; widen later. +- **validation**: + - CI job appears in the workflow. + - At least one PR push triggers the matrix entry and produces + a result (pass or fail) — failure doesn't block merge. +- **status**: Completed +- **log**: + - Audited the existing dispatcher: the env-var path is already + wired as `PIPENV_RESOLVER` (read by + `pipenv.resolver.core._resolver_name_from_env`), honoured by + `_selected_backend_name` after the parent-side CLI / Pipfile + chain falls through. No code change was needed beyond the + workflow file — the plan's aspirational + `PIPENV_RESOLVER_BACKEND` name was renamed to the existing + `PIPENV_RESOLVER` wire to avoid introducing a second env-var + surface (maintenance discipline). + - Smoke-verified: `PIPENV_RESOLVER=pure-python pipenv lock` + produces a byte-identical Pipfile.lock to + `pipenv lock --backend pure-python`. + - Added new job `tests-pure-python-backend` to + `.github/workflows/ci.yaml`: Ubuntu + Python 3.12, + `continue-on-error: true`, modelled on `tests-smoke`'s shape + but with `PIPENV_RESOLVER: pure-python` in the test env. Job + depends on `tests-smoke` so failures don't pre-empt the regular + matrix path. Timeout raised to 45 min for the full-suite run. + - YAML lint clean. +- **files edited/created**: + - `.github/workflows/ci.yaml` (extended) + +--- + +### T_CI2: Update design doc — Phase 3b acceptance gate + +- **depends_on**: `[T_CI1]` +- **location**: `docs/dev/initiative-g-phase3-design.md` (extend + §8 Acceptance Criteria). +- **description**: + Add a Phase 3b row to the acceptance criteria table noting that + the CI matrix is the new strong gate; lockfile byte-identity + on the 100-pkg bench (T15) becomes a sub-criterion of "matrix + passes". The Q-A decision flips from "fail loud" to "build + transparently via PEP 517". Document the + no-build-isolation tradeoff. +- **validation**: + - Doc renders cleanly. + - Phase 3b row added to acceptance matrix. +- **status**: Completed +- **log**: + - Extended `docs/dev/initiative-g-phase3-design.md` §8 with a + Phase 3b sub-section containing: + - Phase 3b acceptance-gate table (resolver-module coverage, + static byte-identity smoke, T15 100-pkg bench, and the new + CI dogfood job) with current status. + - Q-A flip documentation: "fail loud" → "build transparently + via PEP 517", with pointers to T_S1 / T_S2 implementation. + - Q-F survives in repurposed form (T_S4 — emptiness check). + - No-build-isolation tradeoff with win/loss bullets and the + four mitigations actually shipped (path-traversal validation, + device/symlink/fifo rejection, tmp-dir scope, opt-out via + `resolver_backend = "pip"`). + - Updated §9 Out-of-scope: struck through "sdist resolution" + with a back-reference to the Phase 3b row; added "isolated-environment + sdist builds" as the Phase 4 follow-up. +- **files edited/created**: + - `docs/dev/initiative-g-phase3-design.md` (extended) + +--- + +## Parallel Execution Groups + +| Wave | Tasks | Can Start When | +| ---- | ---------------- | ------------------------ | +| 1 | T_M1, T_S1 | Immediately | +| 2 | T_M2, T_S2 | Wave 1 (per-track) | +| 3 | T_M3, T_S3 | Wave 2 (per-track) | +| 4 | T_M4, T_S4 | Wave 3 (per-track) | +| 5 | T_M5, T_S5 | Wave 4 (per-track) | +| 6 | T_S6 | T_S5 done | +| 7 | T_CI1 | T_M5 + T_S6 done | +| 8 | T_CI2 | T_CI1 done | + +Track 1 (markers) and track 2 (sdists) run in parallel for waves 1-5. + +--- + +## Risks & Mitigations + +| Risk | Mitigation | +| ---- | ---------- | +| PEP 517 builds blow up on packages with build-time deps not installed | T_S1 documents the no-build-isolation tradeoff. If a test suite package hits this, fix by pinning build-time deps in pyproject's `requires` (which we already trust the env to satisfy). Phase 3c can add isolation. | +| Marker-string canonicalisation differs from pip's exact format | T_M3 tests assert behaviour, not byte-for-byte string identity. T_PARITY_MATRIX records any format divergence we accept. | +| CI matrix surfaces 20+ failing tests in one go | `continue-on-error: true` is the safety net. Triage one at a time; each fix is its own commit. | +| T_S1 PEP 517 invocation deadlocks on some sdists (build backend never returns) | Wrap the hook call in `concurrent.futures` with a 5-minute timeout; failure surfaces as `SdistBuildError(timeout=...)`. | +| Removing Q-A fail-loud silently masks real failures | T_S4 keeps the "no candidates at all" pre-check so empty-cache cases still surface clearly. | + +--- + +## Out of Scope (Phase 3b) + +Explicitly **not** in Phase 3b: + +- PEP 517 build isolation (Phase 3c if CI surfaces it). +- VCS sources (git/hg/svn URLs in Pipfile). +- File / path sources (`-e .`, local sdists). +- Editable installs (`-e +`). +- Cross-platform lockfiles. +- Promoting the CI matrix to required (separate PR after green). + +If the CI matrix run surfaces a test that needs one of these, log +it as a known-gap in T_PARITY_MATRIX and skip the affected test +under `PIPENV_RESOLVER_BACKEND=pure-python` rather than expanding +Phase 3b scope. + +--- diff --git a/news/+install-drop-redundant-upgrade-flag.behavior.rst b/news/+install-drop-redundant-upgrade-flag.behavior.rst new file mode 100644 index 0000000000..e307731941 --- /dev/null +++ b/news/+install-drop-redundant-upgrade-flag.behavior.rst @@ -0,0 +1,10 @@ +``pip install`` invocations driven by ``pipenv sync`` no longer +pass ``--upgrade``. The flag was hardcoded ``True`` historically as +a safety net to force a re-install when a package was already +present at a different version, but with ``--no-deps`` (always set +for sync) plus explicit ``pkg==X.Y.Z`` lines and the upstream +``Environment.is_satisfied`` filter that already runs before the +batch is handed to pip, ``--upgrade`` is redundant — it forces pip +to do a per-package metadata check that costs measurable wall time +when the lockfile has many entries. ``pipenv install`` +(Pipfile-driven, may need to downgrade) still passes ``--upgrade``. diff --git a/news/+install-parallel-wheel-prefetch.feature.rst b/news/+install-parallel-wheel-prefetch.feature.rst new file mode 100644 index 0000000000..c505921060 --- /dev/null +++ b/news/+install-parallel-wheel-prefetch.feature.rst @@ -0,0 +1,27 @@ +``pipenv install`` and ``pipenv sync`` now pre-fetch wheels in +parallel from the configured indexes BEFORE invoking ``pip install``, +populating a ``--find-links`` directory pip then reads from. Pip's +own install step is sequential, so on a cold pip cache the network +download phase dominates wall time — the sentry-base benchmark's +151 wheels spent ~12 s of pure-network time in pip's subprocess +before this change. Pre-fetching uses the resolver's existing +PEP 691 client + ``ParallelFetcher`` (urllib3 connection-pool capped +at 16 workers, mirroring the established resolver concurrency +contract) and verifies every downloaded body's SHA-256 against the +lockfile's pinned hashes before exposing it to pip. + +The pre-fetch is best-effort: any per-package failure (missing +target-platform wheel, hash mismatch, network hiccup) falls through +silently and pip downloads that package via the index as usual. +The shortcut only fires when the install is driven from a +hash-pinned lockfile (i.e. ``pipenv sync`` or ``pipenv install`` +without ``--skip-lock``); unpinned installs keep the legacy +behaviour. + +Bench impact on the sentry-base fixture (Python 3.11 venv, +151 wheels, cold pip cache): + +* Cold install: ~34 s → ~23 s (~32 % faster, locally) +* Warm install: ~19 s → ~16 s (-3 s — pip's ``--find-links`` + read path is cheaper than its HTTP-cache lookup, so warm-cache + installs benefit too). diff --git a/news/+pipfile-recase-opt-in.behavior.rst b/news/+pipfile-recase-opt-in.behavior.rst new file mode 100644 index 0000000000..ea67587d77 --- /dev/null +++ b/news/+pipfile-recase-opt-in.behavior.rst @@ -0,0 +1,22 @@ +Pipfile name recasing is now opt-in via ``[pipenv] package_name_case``. +Previously, every ``pipenv install`` walked ``packages`` / +``dev-packages`` and made one synchronous PyPI HTTP request per unknown +package name to learn its display capitalization, then rewrote the +Pipfile entry to match. On a fresh ``install -r requirements.txt`` of +~100 packages this added roughly 3 seconds of sequential network +latency on top of the resolve, regardless of cache state. + +The new ``package_name_case`` setting accepts: + +* (unset) / ``"off"`` — default; package names you wrote into your + Pipfile are preserved as-is. PEP 503 normalization still governs + resolution, so behavior is unchanged for matching purposes. +* ``"canonical"`` — apply PEP 503 normalization + (``packaging.utils.canonicalize_name``) to every entry; fully offline. +* ``"pypi"`` — restore the prior behavior, fetching display + capitalization from PyPI per name. + +The toggle lives in the Pipfile rather than in an environment variable +so every contributor on the project gets the same behaviour and +``pipenv install`` does not bounce the Pipfile back and forth between +commits when teammates have different shell defaults. diff --git a/news/+pure-python-extras-roundtrip.bugfix.rst b/news/+pure-python-extras-roundtrip.bugfix.rst new file mode 100644 index 0000000000..0716e64963 --- /dev/null +++ b/news/+pure-python-extras-roundtrip.bugfix.rst @@ -0,0 +1,54 @@ +The pure-Python resolver backend (``[pipenv] resolver_backend = +"pure-python"``) now correctly threads extras +(``pkg[extra1,extra2]``) through the resolver pipeline so +marker-gated transitive dependencies appear in the lockfile. For +example, ``psycopg[binary]`` now pulls ``psycopg-binary`` as a +locked transitive (it didn't before, because the wire-shape parser +dropped the extras section before the resolver ever saw it). + +Five round-trip points were updated together so the bench fixture +keeps converging while T_PARITY_REAL hits 10/10 parity with pip: + +* ``PurePythonBackend._spec_value_to_pipfile_entry`` parses the + ``[extras]`` segment of a wire-shape pip-install line and + surfaces it via the dict-form Pipfile-entry shape that + ``Requirement.from_pipfile_entry`` already accepts. + +* ``PurePythonProvider.find_matches`` clones the filtered candidate + set with ``extras=identifier_extras`` so T7's + ``get_dependencies`` sees the right ``parent_extras`` context + when iterating ``Requires-Dist`` lines. + +* ``PurePythonProvider.get_dependencies`` strips the + ``extra == X`` clauses from a transitive's runtime marker via a + new ``_strip_extra_clauses`` helper. The extras-gating role was + already consumed at emission time; carrying the clause onto + ``Requirement.marker`` made every candidate fail the runtime + marker re-evaluation. ``introducing_marker`` keeps the original + for the lockfile emitter. + +* ``PurePythonProvider.get_dependencies`` also emits a synthetic + base-version requirement when the parent candidate carries + extras — pins the bare ``(name, frozenset())`` identifier to the + exact version of the extras-flavoured candidate. Mirrors pip's + ``ExtrasCandidate.iter_dependencies`` shape and prevents the + bare and extras identifier streams from diverging. + +* ``PurePythonProvider`` now implements + ``narrow_requirement_selection`` (mirroring pip's + ``PipProvider.narrow_requirement_selection``) plus a promote-to- + front polarity flip on ``get_preference``'s leading slot. The + wider transitive constraint graph that extras propagation + surfaces (overlapping upper bounds on protobuf / grpcio / + grpcio-status from the sentry-protos + google-cloud-* fleet) + needed pip-style conflict tracking to converge — without it the + sentry-base bench fixture thrashed indefinitely. The bench now + completes in ~40 s (down from ~2 min baseline) with + byte-identical hash to the pip backend. + +T_PARITY_REAL hits **10/10 byte-identical parity** with the pip +backend across the design's wheel-heavy combos +(``django+psycopg[binary]``, ``flask+gunicorn``, +``fastapi+uvicorn``, ``requests+httpx``, ``pandas+numpy``, +``pytest+pytest-cov``, ``sqlalchemy+alembic``, +``cryptography+pyopenssl``, ``boto3+botocore``, ``click+rich``). diff --git a/news/+pure-python-prerelease-parity.bugfix.rst b/news/+pure-python-prerelease-parity.bugfix.rst new file mode 100644 index 0000000000..de7fb6d100 --- /dev/null +++ b/news/+pure-python-prerelease-parity.bugfix.rst @@ -0,0 +1,14 @@ +The pure-Python resolver backend (``[pipenv] resolver_backend = +"pure-python"``) now mirrors pip's two-pass prerelease filter: the +first pass strictly excludes prereleases (matching pip's +``CandidateEvaluator.get_applicable_candidates`` default), and only +falls back to admitting them when no stable version satisfies the +merged specifier. Previously every prerelease leaked through under +plain ``>=X`` constraints because the per-candidate +``SpecifierSet.contains(..., prereleases=None)`` shape applies +PEP 440's "no final release matched — accept the prerelease" +fallback on a one-element iterable; pip dodges this by filtering the +full candidate list at once. Bench-fixture trigger: ``billiard``, +``hiredis``, and ``sentry-sdk`` were resolving to pre-release +versions (``4.3.0rc1``, ``3.4.0.dev0``, ``3.0.0a7``) where pip +picked the stable below. diff --git a/news/+pure-python-sdist-build-isolation.feature.rst b/news/+pure-python-sdist-build-isolation.feature.rst new file mode 100644 index 0000000000..caf8939833 --- /dev/null +++ b/news/+pure-python-sdist-build-isolation.feature.rst @@ -0,0 +1,14 @@ +The pure-Python resolver backend (``[pipenv] resolver_backend = +"pure-python"``) now builds sdists inside a PEP 517 isolated env via +the (newly-vendored) PyPA :pypi:`build` library. Previously the +sdist METADATA hook ran in pipenv's own interpreter, which crashed +on import for any package whose declared +``[build-system].build-backend`` (e.g. ``poetry-core``, +``hatchling``, ``flit-core``) wasn't already installed. The +resolver now spins up a throwaway venv per build, installs the +project's ``[build-system].requires`` plus any +``get_requires_for_build("wheel")`` extras, then invokes the +backend's ``prepare_metadata_for_build_wheel`` hook against that +env. :pypi:`build` lives at ``pipenv/vendor/build`` so the +resolver subprocess can import it from the project venv's Python +without an extra runtime dep. diff --git a/news/6670.bugfix.rst b/news/6670.bugfix.rst new file mode 100644 index 0000000000..add8008883 --- /dev/null +++ b/news/6670.bugfix.rst @@ -0,0 +1,11 @@ +Restored authentication to private indexes when ``[[source]]`` URLs use +environment-variable placeholders. The GHSA-8xgg-v3jj-95m2 fix moved +credentials off pip's argv onto a merged netrc, but +``write_credentials_netrc`` wrote our Pipfile-derived ``machine`` blocks +BEFORE the appended user netrc — and ``netrc.authenticators()`` returns +the LAST matching entry, so a stale system entry for the same host +silently overrode the freshly-expanded creds. Our blocks now come AFTER +the user's existing content. Additionally, the ``pylock.toml`` reader +now runs ``expand_url_credentials`` over its sources so users with +``[pipenv] use_pylock = true`` see the same env-var expansion that +``Pipfile.lock`` reads have always had. diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index e4ecf569ca..60f0412626 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -329,6 +329,35 @@ def _add_resolver_option(p): ) +def _add_backend_option(p): + """T_PLUMBING (Initiative G phase 3): constrained backend selector. + + User-facing surface for the pluggable resolver-backend dispatcher. + ``--backend NAME`` is the documented spelling on ``pipenv lock``; + accepted values are ``pip`` (default) and ``pure-python``. Unlike + ``--resolver NAME`` (kept for back-compat with T_F.5 scaffolding), + this flag validates the value at parse time via ``choices=`` so + typos fail with an actionable error instead of falling through to + the dispatcher's KeyError → InternalError translation. + + When both flags are supplied, ``--backend`` wins (it's the + constrained form; ``--resolver`` is a free-form passthrough kept + for the older T_F.5 surface). When neither is set, the dispatcher + falls through to PIPENV_RESOLVER / ``[pipenv] resolver_backend`` / + ``[pipenv] resolver`` / ``"pip"``. + """ + p.add_argument( + "--backend", + dest="backend", + choices=("pip", "pure-python"), + default=None, + help=( + "Resolver backend to use. Defaults to " + "[pipenv] resolver_backend in Pipfile, then 'pip'." + ), + ) + + # ── Option group composers ──────────────────────────────────────────────────── @@ -375,6 +404,11 @@ def _add_lock_options(p): _add_dev_option(p, help_text="Generate both develop and default requirements.") _add_dev_only_flag(p) _add_categories_option(p) + # T_PLUMBING (Initiative G phase 3): constrained backend selector. + # ``--backend pure-python`` routes through PurePythonBackend; the + # default ``--backend pip`` (or no flag at all) preserves + # byte-identical pre-T_PLUMBING behaviour. + _add_backend_option(p) def _add_uninstall_options(p): @@ -896,11 +930,17 @@ def build_state(args): state.system = bool(getattr(args, "system", None)) state.verbose = bool(getattr(args, "verbose", None)) state.quiet = bool(getattr(args, "quiet", None)) - # T_F.5: resolver-backend selection. ``None`` means "not specified - # on the CLI"; the dispatcher then falls through to PIPENV_RESOLVER - # / [pipenv] resolver / default. + # T_F.5 / T_PLUMBING: resolver-backend selection. ``None`` means + # "not specified on the CLI"; the dispatcher then falls through to + # PIPENV_RESOLVER / [pipenv] resolver_backend / [pipenv] resolver + # / default. The constrained ``--backend NAME`` flag (T_PLUMBING, + # Initiative G phase 3) takes precedence over the free-form + # ``--resolver NAME`` flag (T_F.5 scaffolding) when both are + # supplied — ``--backend`` is the documented user-facing surface. + raw_backend = getattr(args, "backend", None) raw_resolver = getattr(args, "resolver", None) - state.resolver = raw_resolver.strip() if isinstance(raw_resolver, str) and raw_resolver.strip() else None + chosen = raw_backend if raw_backend else raw_resolver + state.resolver = chosen.strip() if isinstance(chosen, str) and chosen.strip() else None # ── Validation ─────────────────────────────────────────────────────────── if state.python: diff --git a/pipenv/resolver/backends/__init__.py b/pipenv/resolver/backends/__init__.py index 82491b70b6..73e4aac656 100644 --- a/pipenv/resolver/backends/__init__.py +++ b/pipenv/resolver/backends/__init__.py @@ -18,6 +18,7 @@ from pipenv.resolver.backends.base import REGISTRY, Backend from pipenv.resolver.backends.pip import PipBackend +from pipenv.resolver.backends.pure_python import PurePythonBackend # Default backend used when no explicit selection has been made via CLI, # env var, or Pipfile. Kept at module scope so the dispatcher in @@ -29,6 +30,12 @@ # monkey-patch via ``mock.patch.dict`` see the same dict no matter # which module they reach for. REGISTRY["pip"] = PipBackend +# Initiative G phase 3 T10: register the pure-Python resolver backend. +# Pip stays the default — see ``DEFAULT_BACKEND_NAME`` above. The +# pure-python class is zero-arg-constructible (collaborators default to +# ``None``); the dispatcher injects cache/fetcher/session/metadata_cache +# before invoking ``.resolve()`` (Phase 4 wiring). +REGISTRY["pure-python"] = PurePythonBackend def get_backend(name: str) -> Backend: @@ -80,6 +87,7 @@ def list_backends() -> list[str]: "Backend", "DEFAULT_BACKEND_NAME", "PipBackend", + "PurePythonBackend", "REGISTRY", "get_backend", "list_backends", diff --git a/pipenv/resolver/backends/pure_python.py b/pipenv/resolver/backends/pure_python.py new file mode 100644 index 0000000000..25c5dc668d --- /dev/null +++ b/pipenv/resolver/backends/pure_python.py @@ -0,0 +1,841 @@ +"""``pure-python`` resolver backend (Initiative G Phase 3, T9). + +Implements the Initiative F ``Backend`` protocol over the +:class:`pipenv.resolver.pure_python_provider.PurePythonProvider` (T3-T8) ++ :func:`pipenv.resolver.pure_python_metadata.fetch_metadata` (T2) + +:class:`pipenv.resolver.pure_python_requirement.Requirement` (T1) +chain. + +Flow inside :meth:`PurePythonBackend.resolve`: + +1. **Pre-fetch** top-level package candidates via + :meth:`ParallelFetcher.populate` (Q-B). +2. **Top-level emptiness pre-check** (Phase 3b T_S4): scan cached + candidates for each top-level package; abort with a structured + :class:`ResolutionError` if any top-level has ZERO candidates + across every configured index. Catches typos, yanked-only + releases, and cold-cache + total-fetch-failure cases at startup + instead of 30 s into a doomed resolve. Phase 3a's older + wheel-availability variant of this gate (which fired on + sdist-only top-levels) is gone — T_S2/T_S3 build sdists + transparently so sdist-only top-levels resolve normally. +3. Build :class:`Requirement` set from ``request.packages.specs``. +4. Drive :class:`resolvelib.Resolver` via + :func:`pipenv.resolver.pure_python_provider._drive_resolver`; any + sdist METADATA needed during expansion is built transparently via + T_S2's :meth:`MetadataFetcher.fetch_metadata` routing (Phase 3b + T_S3 removed the Phase 3a ``_SdistEncountered`` handler — sdists + no longer reach an error path). +5. :class:`resolvelib.ResolutionImpossible` → + :class:`ResolutionError`; any other exception → + :class:`InternalError`; otherwise translate the resolved candidate + mapping into a tuple of :class:`LockedRequirement` and return + :class:`ResolverSuccess`. + +See ``docs/dev/initiative-g-phase3-design.md`` §5.4 and +``initiative-g-phase3-plan.md`` T9. + +NOTE: this module imports only from ``pipenv.patched.pip._vendor.resolvelib`` +(vendor, not internal — that's the resolvelib exception type lookup +inside :meth:`resolve`). The pipenv pure-Python backend deliberately +sits on the typed schema surface, not on patched-pip internals. +""" +from __future__ import annotations + +import os +import traceback +from pathlib import Path +from typing import Any, Sequence + +from pipenv.resolver.pure_python_metadata import ( + MetadataCache, + fetch_metadata, +) +from pipenv.resolver.pure_python_provider import ( + PurePythonProvider, + _drive_resolver, +) +from pipenv.resolver.pure_python_requirement import Requirement +from pipenv.resolver.schema import ( + SCHEMA_VERSION, + Diagnostics, + InternalError, + LockedRequirement, + ResolutionError, + ResolverRequest, + ResolverResponse, + ResolverSuccess, +) +from pipenv.vendor.packaging.specifiers import ( + InvalidSpecifier, + SpecifierSet, +) + + +class PurePythonBackend: + """Resolver backend that drives the pure-Python resolvelib provider. + + Attributes + ---------- + cache: + :class:`pipenv.resolver.manifest_cache.ParsedManifestCache` (or + compatible) — the candidate-listing store the provider's + :meth:`find_matches` reads. + fetcher: + :class:`pipenv.resolver.fetcher.ParallelFetcher` (or compatible) + — used to populate the cache for the top-level package set + before resolution starts. + session: + Duck-typed HTTP session passed verbatim to + :func:`fetch_metadata`. Production hands us a configured + ``urllib3.PoolManager``; tests pass a :class:`mock.MagicMock`. + metadata_cache: + :class:`MetadataCache` used as the read-through cache for + wheel-METADATA bodies. + target_env: + Optional mapping of marker-environment variables (``python_version``, + ``sys_platform``, etc.). Defaults to a snapshot of the running + interpreter's :func:`packaging.markers.default_environment` + (added lazily on first use). + index_urls: + Tuple of simple-API index URLs to consult for candidates. + """ + + name: str = "pure-python" + + def __init__( + self, + *, + cache: Any = None, + fetcher: Any = None, + session: Any = None, + metadata_cache: MetadataCache | Any = None, + target_env: dict | None = None, + index_urls: Sequence[str] = ("https://pypi.org/simple",), + ) -> None: + # All collaborators default to ``None`` so the class is + # zero-arg-constructible — the Initiative F registry's + # :func:`pipenv.resolver.backends.get_backend` helper invokes + # ``cls()`` for class-shaped registry entries (see T10). A + # backend created without collaborators answers ``.name`` and + # ``.is_available()`` correctly; ``.resolve()`` requires the + # dispatcher to inject collaborators (production wiring lives + # in the Phase 4 dispatcher work). Tests pass collaborators + # explicitly via keyword args. + self._cache = cache + self._fetcher = fetcher + self._session = session + self._metadata_cache = metadata_cache + self._target_env = target_env + self._index_urls: tuple[str, ...] = tuple(index_urls) + + # ------------------------------------------------------------------ + # Backend protocol + # ------------------------------------------------------------------ + + def is_available(self) -> bool: + """The pure-python backend is always available — it ships + in-tree as part of ``pipenv.resolver``; there is no external + binary or optional dependency to probe for. + """ + return True + + def resolve(self, request: ResolverRequest) -> ResolverResponse: + """Run resolution against the typed request envelope. + + Returns a typed :class:`ResolverResponse`. Failure modes are + translated to structured ``result`` variants rather than raised + out of the backend (per the Initiative F ``Backend`` contract). + """ + # Bootstrap collaborators from the request when not pre-injected + # (T9b, 2026-05-12). Production code arrives here via the + # Initiative F registry's ``get_backend("pure-python")`` helper + # which calls ``cls()`` with no args (T10) — so the four + # collaborators are ``None`` on ``self``. Unit tests still + # inject them via ``__init__`` kwargs; the bootstrap is + # idempotent and only fills fields that are currently ``None``. + self._bootstrap_from_request(request) + + # --- index URLs from request.sources ---------------------------- + # request.sources is the post-mirror-substitution source list; + # if empty (subprocess fixtures sometimes omit it) fall through + # to the backend's configured default. + request_index_urls = tuple(s.url for s in request.sources) or self._index_urls + + # T_M4 (Initiative G Phase 3b): URL→name map for the lockfile + # ``index`` field. Pip writes ``index=`` (e.g. + # ``"pypi"``) per the Pipfile ``[[source]]`` block, NOT the raw + # URL. We build the map here so the translator can look up the + # source name for each resolved candidate. Empty when + # ``request.sources`` is empty (no mapping possible — the + # translator falls back to emitting the URL verbatim). + url_to_name: dict[str, str] = { + s.url: s.name for s in request.sources + } + + # Top-level names per the typed schema: + # ``request.packages.specs`` is a Mapping[str, str]. + top_level_names = tuple(sorted(request.packages.specs.keys())) + + # ---- Step 1: pre-fetch (Q-B) ----------------------------------- + targets = [ + (idx, name) + for idx in request_index_urls + for name in top_level_names + ] + try: + self._fetcher.populate(targets) + except Exception: # noqa: BLE001 + # Pre-fetch failures are non-fatal in production (Q-B says + # the provider falls through to lazy ``populate`` on cache + # miss). Surface the error to diagnostics-style behaviour + # but continue — let ``find_matches`` retry on miss. The + # top-level emptiness pre-check below catches the case + # where pre-fetch *and* every lazy retry leave the cache + # empty for some top-level (typo / yanked-only / total + # network blackout); resolvelib's own error path covers + # the "candidates exist but none satisfy" case. + pass + + # ---- Step 2: top-level emptiness pre-check (T_S4) -------------- + # For each top-level package, scan cached candidates across + # configured indexes. If a top-level has ZERO candidates on + # *every* configured index → fail loud BEFORE driving + # resolvelib. This catches typos, yanked-only releases, and + # cold-cache + every-fetch-failed cases with a clear message + # instead of letting resolvelib raise its own opaque + # "no candidates available" error 30 s later. + # + # NB Phase 3a fired this gate on sdist-only top-levels too; + # T_S2 (sdists build via PEP 517) and T_S3 (no fail-loud on + # sdist encounter) made that branch obsolete and it was + # removed in T_S4 — sdist-only top-levels now resolve + # normally. + empty_top_level: list[str] = [] + for name in top_level_names: + saw_any = False + for idx in request_index_urls: + manifest = self._cache.get(idx, name) + if manifest is None: + continue + if any(True for _ in getattr(manifest, "candidates", ())): + saw_any = True + break + if not saw_any: + empty_top_level.append(name) + + if empty_top_level: + names = sorted(empty_top_level) + index_plural = "es" if len(request_index_urls) > 1 else "" + pip_message = ( + f"Pure-python backend cannot resolve {names!r}: no candidates " + f"found in the configured index{index_plural}. " + f"Check the package name (typo?), confirm releases on the index, " + f"or retry with `pipenv lock --backend pip` if the package is " + f"available through pip's resolver but not the simple-API." + ) + return ResolverResponse( + schema_version=SCHEMA_VERSION, + result=ResolutionError( + kind="resolution_error", + conflicts=(), + pip_message=pip_message, + ), + diagnostics=Diagnostics(), + ) + + # ---- Step 3: build Requirement set ----------------------------- + requirements: list[Requirement] = [] + for name, spec_value in request.packages.specs.items(): + # The wire-shape ``spec_value`` is a pip-install argument + # line (e.g. ``"requests==2.31.0"``). Pipfile entries flow + # through here too — Requirement.from_pipfile_entry accepts + # str values (parsed as a SpecifierSet directly) and dict + # values. Handle both shapes. + requirements.append( + Requirement.from_pipfile_entry( + name, + self._spec_value_to_pipfile_entry(name, spec_value), + ) + ) + + # ---- Step 4: construct provider -------------------------------- + # Bind a metadata-fetcher closure around T2's fetch_metadata so + # the provider's get_dependencies (T7) receives a callable that + # only takes a Candidate. + session = self._session + metadata_cache = self._metadata_cache + + def _metadata_fetcher(candidate: Any): + return fetch_metadata(candidate, session, cache=metadata_cache) + + provider = PurePythonProvider( + cache=self._cache, + fetcher=self._fetcher, + metadata_fetcher=_metadata_fetcher, + target_env=self._resolved_target_env(), + index_urls=request_index_urls, + allow_prereleases=bool(request.options.pre), + ) + + # ---- Step 5: drive resolvelib ---------------------------------- + # Local import to avoid module-load-time cost for callers who + # never select this backend. + from pipenv.patched.pip._vendor.resolvelib import ResolutionImpossible + + try: + result = _drive_resolver(requirements, provider) + except ResolutionImpossible as exc: + pip_message = self._format_resolution_impossible(exc) + return ResolverResponse( + schema_version=SCHEMA_VERSION, + result=ResolutionError( + kind="resolution_error", + # ConflictRecord has fields {package, version, + # requires} — these are pip-style English-table + # rows, not resolvelib's RequirementInformation + # shape. We leave this empty (the pip_message + # carries the same information in a free-text + # form) rather than synthesise an awkward + # translation that would diverge from the pip + # backend's format. + conflicts=(), + pip_message=pip_message, + ), + diagnostics=Diagnostics(), + ) + except Exception as exc: # noqa: BLE001 + return ResolverResponse( + schema_version=SCHEMA_VERSION, + result=InternalError( + kind="internal_error", + message=str(exc), + traceback=traceback.format_exc(), + ), + diagnostics=Diagnostics(), + ) + + # ---- Step 6: translate resolved mapping ------------------------ + # T_M3 (Initiative G Phase 3b): the translator now reads the + # ``criteria`` side-channel from the resolvelib ``Result`` to + # emit ``markers`` clauses, so we hand it the full Result rather + # than just ``.mapping``. + # T_M4 (Initiative G Phase 3b): thread the URL→name map so the + # translator can emit the source NAME for the ``index`` field + # instead of the URL (pip-parity). + locked = self._translate_mapping( + result, request_index_urls, url_to_name + ) + return ResolverResponse( + schema_version=SCHEMA_VERSION, + result=ResolverSuccess(kind="success", locked=locked), + diagnostics=Diagnostics(), + ) + + # ------------------------------------------------------------------ + # Bootstrap (T9b — 2026-05-12) + # ------------------------------------------------------------------ + + def _bootstrap_from_request(self, request: ResolverRequest) -> None: + """Populate missing collaborators from the request envelope. + + Idempotent: only fills fields that are currently ``None`` on + ``self``. Tests that pass collaborators via ``__init__`` + kwargs are unaffected — their pre-injected values short-circuit + every branch below. + + Production wiring (the Initiative F registry path) lands here + with all four fields at ``None`` because + :func:`pipenv.resolver.backends.get_backend` invokes ``cls()`` + with no args (T10). Constructor signatures are pinned to the + existing modules: + + * :class:`pipenv.resolver.pep691.PEP691Client` — ``(session, + *, netrc_path, cert, verify)``. + * :class:`pipenv.resolver.fetcher.ParallelFetcher` — ``(client, + cache, *, max_workers, default_ttl)``. + * :class:`pipenv.resolver.manifest_cache.ParsedManifestCache` — + ``(root, schema_version)``. + * :class:`pipenv.resolver.pure_python_metadata.MetadataCache` — + ``(root)``. + + Cache-dir resolution mirrors the production prefetch path at + ``pipenv/routines/lock.py::_prefetch_index_manifests_if_enabled``: + ``$PIPENV_CACHE_DIR / "pipenv-manifests"`` for the manifest + cache root. ``ResolverRequest`` carries no ``cache_dir`` field + of its own (see ``pipenv/resolver/schema.py``), so we fall back + to the environment variable with a user-cache default — this + matches the convention :class:`pipenv.environments.Setting` + uses to resolve :attr:`PIPENV_CACHE_DIR` on the parent side. + + ``verify_ssl`` is taken from the *first* source on the request + (Phase 3 single-policy simplification). Multi-policy fan-out + across heterogeneous source sets is the lock-route's + responsibility — when the dispatcher constructs the backend + with collaborators pre-built, the FU2 per-policy fan-out at + ``lock.py`` applies and this branch is skipped entirely. + """ + # Session: a urllib3 PoolManager (production) or a duck-typed + # mock (tests). We follow ``pipenv.utils.internet.get_requests_session`` + # — the same factory the production prefetch path uses — so + # cert/verify/cache-dir wiring is shared between the prefetcher + # and this backend. Failure here is *not* fatal: if pip's + # internals aren't importable for any reason (e.g. a sandboxed + # test path), fall back to a bare PoolManager and surface the + # error downstream via the fetcher's own error envelope. + if self._session is None: + verify = bool(request.sources[0].verify_ssl) if request.sources else True + try: + from pipenv.utils.internet import get_requests_session + + self._session = get_requests_session(verify_ssl=verify) + except Exception: # noqa: BLE001 — last-resort fallback. + from pipenv.patched.pip._vendor.urllib3 import PoolManager + + self._session = PoolManager() + + # Manifest cache: filesystem-backed, rooted under PIPENV_CACHE_DIR. + if self._cache is None: + from pipenv.resolver.manifest_cache import ParsedManifestCache + + cache_root = self._cache_dir_from_request(request) / "pipenv-manifests" + self._cache = ParsedManifestCache(cache_root) + + # Parallel fetcher: needs a PEP691Client wrapping the session + + # the manifest cache built above. ``verify`` is sourced from + # the first request source (Phase 3 single-policy default). + if self._fetcher is None: + from pipenv.resolver.fetcher import ParallelFetcher + from pipenv.resolver.pep691 import PEP691Client + + verify = bool(request.sources[0].verify_ssl) if request.sources else True + client = PEP691Client(self._session, verify=verify) + self._fetcher = ParallelFetcher(client, self._cache) + + # Metadata cache: separate filesystem cache (different on-disk + # schema vs. manifest cache, so a sibling directory rather than + # nesting under ``pipenv-manifests``). + if self._metadata_cache is None: + metadata_root = ( + self._cache_dir_from_request(request) + / "pipenv-manifests" + / "metadata-v1" + ) + self._metadata_cache = MetadataCache(metadata_root) + + @staticmethod + def _cache_dir_from_request(request: ResolverRequest) -> Path: + """Resolve the on-disk cache root. + + ``ResolverRequest`` does not carry a ``cache_dir`` field today; + we mirror the parent-side default chain used by + :class:`pipenv.environments.Setting`:: + + $PIPENV_CACHE_DIR + → pip's USER_CACHE_DIR (fallback to ~/.cache/pipenv on + POSIX, %LOCALAPPDATA%/pipenv/Cache on Windows) + + We deliberately do NOT import pip-internal locations here — + that's the coupling Initiative G exists to break. The + ``~/.cache/pipenv`` fallback is the Linux default of pip's + USER_CACHE_DIR and is good enough for the bootstrap path; + production runs always have ``PIPENV_CACHE_DIR`` set by the + parent. + """ + env_value = os.environ.get("PIPENV_CACHE_DIR") + if env_value: + return Path(env_value) + return Path.home() / ".cache" / "pipenv" + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + @staticmethod + def _spec_value_to_pipfile_entry(name: str, spec_value: str) -> Any: + """Translate a wire-shape pip-install line into a value that + :meth:`Requirement.from_pipfile_entry` understands. + + The wire format is a full pip argument string like + ``"requests==2.31.0"``, ``"urllib3[brotli]>=2 -i https://…"``, + ``"psycopg[binary]"``, or just the package name (``"requests"`` + ⇒ "any version"). Pip flags (``-i``, ``--index-url``, + ``--trusted-host``, etc.) appear after a whitespace separator; + the backend handles index URLs via + ``request.options.indexes``, so we drop everything past the + first whitespace token before extracting the specifier. + + Return shape: a plain specifier string when the wire-shape + carries no extras (preserves backwards compatibility with the + existing test contract), or a dict with ``version`` + ``extras`` + keys when ``name[extra1,extra2]`` is present. The Pipfile + parser at :meth:`Requirement.from_pipfile_entry` accepts both + shapes — dict-form is how it normally consumes + ``pkg = { version = "*", extras = [...] }`` from a TOML file. + + T_PARITY_REAL bench trigger (project 01 — ``psycopg[binary]``): + before this fix, the wire-shape ``"psycopg[binary]"`` returned + ``"*"`` outright (the bracketed extras section contains no + specifier-marker character, so the loop fell through), which + silently dropped the extras from the resulting + :class:`Requirement` and made the resolvelib identifier + ``("psycopg", frozenset())`` rather than + ``("psycopg", frozenset({"binary"}))``. Without the right + identifier, T7's :meth:`get_dependencies` evaluated + ``Requires-Dist`` markers under ``extra=""`` and the + ``psycopg-binary; extra == "binary"`` transitive disappeared. + """ + line = (spec_value or "").strip() + if not line: + return "*" + # First whitespace-separated token is the + # ``name[extras]`` part; everything after is pip + # CLI flags which the backend doesn't consume here. + first_token = line.split(None, 1)[0] + # Strip ``[extras]`` from the name segment (if present) BEFORE + # scanning for specifier markers; otherwise the inner bracket + # contents (e.g. ``security,brotli``) can confuse the scan and + # we'd lose the extras anyway. + extras: list[str] = [] + bracket_open = first_token.find("[") + if bracket_open != -1: + bracket_close = first_token.find("]", bracket_open + 1) + if bracket_close != -1: + extras = [ + e.strip() + for e in first_token[bracket_open + 1 : bracket_close].split(",") + if e.strip() + ] + # Rejoin around the bracket span so the specifier scan + # below works on a flat ``name`` shape. + first_token = ( + first_token[:bracket_open] + first_token[bracket_close + 1 :] + ) + specifier = "*" + for marker_char in ("==", ">=", "<=", "~=", "!=", ">", "<"): + idx = first_token.find(marker_char) + if idx != -1: + specifier = first_token[idx:] + break + if extras: + return {"version": specifier, "extras": extras} + return specifier + + def _resolved_target_env(self) -> dict: + """Return a marker-environment dict for the provider. + + Computed lazily so test fixtures that pass an explicit + ``target_env`` win, and production paths get the running + interpreter's defaults without paying the cost on every + ``resolve`` call from a long-lived process. + """ + if self._target_env is not None: + return self._target_env + # Default to the running interpreter's marker environment. + # ``packaging.markers.default_environment`` is the canonical + # PEP 508 source; we route through the vendored copy so this + # module avoids any patched-pip-internal coupling. + from pipenv.vendor.packaging.markers import default_environment + + env = dict(default_environment()) + self._target_env = env + return env + + @staticmethod + def _format_resolution_impossible(exc: Any) -> str: + """Render a :class:`ResolutionImpossible` as a multi-line + ``pip_message`` body. + + Each cause is rendered as ``" - requires "`` + so the user can locate the conflict without parsing resolvelib + internals. + """ + lines = ["Resolution impossible — conflicting constraints:"] + causes = getattr(exc, "causes", None) or () + for info in causes: + req = getattr(info, "requirement", None) + parent = getattr(info, "parent", None) + parent_label: str + if parent is None: + parent_label = "" + else: + p_name = getattr(parent, "name", None) + p_version = getattr(parent, "version", None) + if p_name and p_version: + parent_label = f"{p_name} {p_version}" + else: + parent_label = str(parent) + req_name = getattr(req, "name", "") + req_spec = getattr(req, "specifier", "") + lines.append(f" - {parent_label} requires {req_name}{req_spec}") + return "\n".join(lines) + + def _translate_mapping( + self, + result: Any, + index_urls: Sequence[str], + url_to_name: dict[str, str] | None = None, + ) -> tuple[LockedRequirement, ...]: + """Translate the resolvelib ``Result`` into typed + :class:`LockedRequirement` entries. + + ``result`` is the :class:`resolvelib.resolvers.abstract.Result` + namedtuple ``(mapping, graph, criteria)``. Today we read + ``mapping`` (resolved candidates) and ``criteria`` (per-identifier + :class:`Criterion` whose ``.information`` is the list of + :class:`RequirementInformation` rows that selected each candidate). + + Field mapping (per :class:`LockedRequirement` in + ``pipenv/resolver/schema.py``): + + * ``name`` ← ``candidate.name`` + * ``version`` ← ``"==" + candidate.version`` (matches the + existing pip-backend lockfile shape — see + ``Entry.get_cleaned_dict`` / ``_clean_version``). + * ``extras`` ← ``tuple(sorted(candidate.extras))`` + * ``markers`` ← ``and``-join of: + - the Requires-Python marker derived from + ``candidate.requires_python`` (T_M3), and + - the ``or``-join of every ``Requirement.introducing_marker`` + attached to the criterion's ``information`` rows (T_M2 + + T_M3). + ``None`` when neither source contributes. See + :func:`_requires_python_to_marker` and + :func:`_introducing_marker_for` for the per-source rules. + * ``hashes`` ← ``tuple(sorted(":"))`` unioned + from every cache sibling whose ``version`` matches the + resolved candidate's, across every configured index URL + (T_S5 — pip-parity: lockfile lists hashes for ALL distfiles + of the resolved version, not just the chosen candidate). + Falls back to ``candidate.hashes`` alone when the cache has + no siblings (cold-cache / test-fixture-injected workflows). + * ``index`` ← source NAME (e.g. ``"pypi"``) when + ``default_index`` (the first URL in ``index_urls``) appears + in ``url_to_name``; otherwise the URL is emitted verbatim as + a defensive fallback (Phase 3b simplification — Phase 4 will + track which configured source actually served each candidate + via per-Candidate provenance). + + Backward-compat: callers that hand us a bare ``.mapping`` dict + (older test fixtures pre-T_M3 used ``_FakeResult(mapping=...)`` + without criteria) get ``criteria = {}`` and markers fall back + to the Requires-Python contribution only. Callers that hand + us a plain ``dict`` (no ``.mapping`` attribute) are also + supported — we treat it as the mapping directly. Callers that + omit ``url_to_name`` get an empty map ⇒ all candidates fall + through to the URL-verbatim fallback (back-compat for legacy + fixtures from before T_M4 introduced the third parameter). + """ + # Default index for the lockfile entries — Phase 3 takes the + # first configured URL. + default_index = index_urls[0] if index_urls else None + # T_M4: empty mapping when callers omit ``url_to_name`` (older + # test fixtures and the pre-T_M4 signature). An empty map ⇒ + # ``dict.get`` always falls back to the URL verbatim, preserving + # the pre-T_M4 behaviour for those callers. + url_to_name = url_to_name or {} + + # Tolerate both the resolvelib ``Result`` shape (with ``.mapping`` + # and ``.criteria``) and a bare-dict mapping (pre-T_M3 tests). + mapping = getattr(result, "mapping", result) + criteria = getattr(result, "criteria", {}) or {} + + locked: list[LockedRequirement] = [] + for identifier, candidate in mapping.items(): + # identifier is (name, frozenset(extras)) per T3. + try: + _name, extras_set = identifier + except (TypeError, ValueError): + _name = getattr(candidate, "name", str(identifier)) + extras_set = frozenset() + + cand_name = getattr(candidate, "name", _name) + cand_version = getattr(candidate, "version", None) + if cand_version is None: + # Skip non-versioned candidates defensively — resolvelib + # should never produce one but a malformed test fixture + # would otherwise crash the LockedRequirement constructor. + continue + + # Hash translation (T_S5 — Initiative G Phase 3b): + # mirror pip's lockfile convention by emitting hashes for + # EVERY distfile of the resolved version (wheel + sdist + + # any cross-platform wheel variants), not just the chosen + # candidate's single hash. Walk every configured index and + # union the ``hashes`` frozensets of cache siblings whose + # ``version`` matches the resolved one. + # + # Dedup is set-driven: identical ``(algo, value)`` pairs + # served by multiple indexes collapse to one entry. + # + # Fallback (defensive): when no sibling-candidate scan + # turned up anything (cold cache / a test fixture that + # injects candidates straight into the result without + # populating the cache), fall back to the resolved + # candidate's own ``hashes``. Preserves Phase 3a + # fixture-injected-only workflows. + hashes_iter: set[str] = set() + for idx in index_urls: + manifest = self._cache.get(idx, cand_name) + if manifest is None: + continue + for sibling in getattr(manifest, "candidates", ()) or (): + if getattr(sibling, "version", None) != cand_version: + continue + for h in getattr(sibling, "hashes", frozenset()) or (): + algo = getattr(h, "algo", None) + value = getattr(h, "value", None) + if algo and value: + hashes_iter.add(f"{algo}:{value}") + if not hashes_iter: + for h in getattr(candidate, "hashes", frozenset()) or (): + algo = getattr(h, "algo", None) + value = getattr(h, "value", None) + if algo and value: + hashes_iter.add(f"{algo}:{value}") + hashes_tuple = tuple(sorted(hashes_iter)) + + extras_tuple = tuple(sorted(extras_set or frozenset())) + + # T_M3 — marker emission. + requires_python_marker = _requires_python_to_marker( + getattr(candidate, "requires_python", None) + ) + criterion = criteria.get(identifier) + criterion_information = ( + getattr(criterion, "information", ()) if criterion is not None else () + ) + introducing_marker = _introducing_marker_for(criterion_information) + marker_string = _combine_markers( + [requires_python_marker, introducing_marker] + ) + + # T_M4: emit source NAME when the URL maps to a configured + # source; fall back to the URL verbatim if no source matches + # (defensive — the Phase 3 Candidate doesn't track WHICH + # configured source served it, so this lookup is approximate. + # Phase 4 will track per-candidate index provenance for + # exact attribution across multi-source resolutions). + index_value = ( + url_to_name.get(default_index, default_index) + if default_index is not None + else None + ) + + locked.append( + LockedRequirement( + name=cand_name, + version=f"=={cand_version}", + extras=extras_tuple, + markers=marker_string, + hashes=hashes_tuple, + index=index_value, + ) + ) + + return tuple(locked) + + +# --------------------------------------------------------------------------- +# T_M3 — marker translation helpers (Initiative G Phase 3b) +# --------------------------------------------------------------------------- + + +def _requires_python_to_marker(requires_python: str | None) -> str | None: + """Translate a ``Requires-Python`` specifier-set string into a + canonical marker string. + + Examples:: + + ">=3.10" → "python_version >= '3.10'" + ">=3.8,<4" → "python_version < '4' and python_version >= '3.8'" + None / "" → None + unparseable → None (defensive — mirrors T4's behaviour for + malformed ``Requires-Python`` strings on + index manifests) + + Specs are joined in sorted lexicographic order so the emitted + marker string is stable across runs (resolvelib doesn't guarantee + spec-set iteration order — :class:`SpecifierSet` is backed by a + ``frozenset``). + """ + if not requires_python: + return None + try: + spec_set = SpecifierSet(requires_python) + except (InvalidSpecifier, ValueError): + return None + parts: list[str] = [] + for spec in spec_set: + op = spec.operator # one of '==', '!=', '<=', '>=', '<', '>', '~=', '===' + ver = spec.version + parts.append(f"python_version {op} {ver!r}") + if not parts: + return None + # Stable order across runs: ``SpecifierSet`` iterates a frozenset + # under the hood, so iteration order is unspecified. Sort here so + # the lockfile output is byte-identical on every invocation. + parts.sort() + return " and ".join(parts) + + +def _introducing_marker_for(criterion_information: Any) -> str | None: + """OR-combine ``introducing_marker`` strings from every requirement + that selected the candidate (T_M2 populates the slot on each + transitive :class:`Requirement`). + + Rationale for the OR-join (with parentheses on each clause): + a candidate is admitted to the lockfile because *every* introducing + requirement was active in the resolver's environment. At install + time, the candidate is needed if ANY of those requirements is + active. The lockfile's ``markers`` clause therefore encodes the + union of preconditions — a logical ``or``. Parentheses preserve + precedence when a downstream consumer AND-merges this with another + marker clause (e.g. the Requires-Python contribution combined in + :meth:`_translate_mapping`). + + Empty / all-None ``information`` → returns ``None`` (no + contribution). Single non-None marker → its string verbatim (no + parens — readability over uniformity, matches what pip emits). + Duplicates (same marker string from multiple parents) are + deduplicated, preserving insertion order so the output is + fixture-stable. + """ + markers_seen: list[str] = [] + for info in criterion_information or (): + req = getattr(info, "requirement", None) + intro = getattr(req, "introducing_marker", None) + if intro is None: + continue + text = str(intro).strip() + if not text: + continue + if text in markers_seen: + continue + markers_seen.append(text) + if not markers_seen: + return None + if len(markers_seen) == 1: + return markers_seen[0] + return " or ".join(f"({m})" for m in markers_seen) + + +def _combine_markers(parts: Sequence[str | None]) -> str | None: + """AND-join non-None marker strings. Returns ``None`` when every + contribution is ``None`` / empty. + + Single non-None part → that part verbatim (no surrounding + parens — keeps the simple ``python_version >= '3.10'`` form for the + common "Requires-Python only" case). Multiple parts → joined with + ``" and "`` in the supplied order (caller controls ordering; the + translator passes Requires-Python first then introducing markers). + """ + non_none = [p for p in parts if p] + if not non_none: + return None + if len(non_none) == 1: + return non_none[0] + return " and ".join(non_none) + + +__all__ = ["PurePythonBackend"] diff --git a/pipenv/resolver/candidate.py b/pipenv/resolver/candidate.py index c7614923db..9d6ea53695 100644 --- a/pipenv/resolver/candidate.py +++ b/pipenv/resolver/candidate.py @@ -22,7 +22,7 @@ """ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime from typing import Any, NamedTuple @@ -101,6 +101,17 @@ class Candidate: :func:`pipenv.vendor.packaging.tags.parse_tag` so a compatibility check is a frozenset intersection (vs pip's ``Wheel.supported(tags)`` linear scan). + extras: + ``frozenset[str]`` of extras this candidate provides, used by + :meth:`PurePythonProvider.identify` (T3) to compute the + ``(name, extras)`` graph identifier. The PEP 691 simple-API + does NOT report extras on candidates — the field exists so + callers that want to model "this candidate satisfies the + ``[argon2]`` extra of django" (e.g. constructed in unit tests + or by future code that synthesises extras-bearing candidates) + can do so. Default is ``frozenset()`` — the PEP 691 parsers + in :mod:`pipenv.resolver.manifest_cache` rely on this default + and don't need to plumb the field through. """ name: str @@ -114,6 +125,11 @@ class Candidate: upload_time: datetime | None is_wheel: bool wheel_tags: frozenset[Tag] | None + # ``extras`` is added in Initiative G phase 3 (T3) — Phase 1's PEP + # 691 parsers don't populate it (the simple-API doesn't report + # extras), so it defaults to an empty frozenset. Placed last so + # frozen-dataclass field ordering (no-default-then-default) holds. + extras: frozenset[str] = field(default_factory=frozenset) @classmethod def from_filename(cls, __filename: str, /, **kwargs: Any) -> Candidate: diff --git a/pipenv/resolver/core.py b/pipenv/resolver/core.py index 62fcb686e2..a9f3f31405 100644 --- a/pipenv/resolver/core.py +++ b/pipenv/resolver/core.py @@ -641,7 +641,15 @@ def _resolver_name_from_pipfile() -> str | None: return None try: project = Project(chdir=False) - resolver = project.settings.get("resolver") + # T_PLUMBING (Initiative G phase 3): prefer the documented + # ``resolver_backend`` key over the T_F.5 back-compat alias + # ``resolver``. Either may be present; the documented spelling + # wins to match the parent-side precedence in + # :func:`pipenv.routines.lock.do_lock`. + resolver = ( + project.settings.get("resolver_backend") + or project.settings.get("resolver") + ) except Exception: # noqa: BLE001 — any read failure → default return None if not resolver: diff --git a/pipenv/resolver/pep691.py b/pipenv/resolver/pep691.py index 760a91aac1..a974fd9581 100644 --- a/pipenv/resolver/pep691.py +++ b/pipenv/resolver/pep691.py @@ -921,14 +921,35 @@ def _dispatch_response( target_url: str, if_none_match: str | None, ) -> SimplePageResponse | FetchError: - """Branch on HTTP status; parse body for 200; map errors.""" - status = response.status + """Branch on HTTP status; parse body for 200; map errors. + + Accepts both urllib3-style responses (``.status`` / ``.data``) + and ``requests``-style responses (``.status_code`` / ``.content``). + Production uses a :class:`PipSession` (``requests``-based); the + Phase 1+2 unit tests use a urllib3-style mock — picking by + attribute *presence* (not value) keeps urllib3-style mocks with + an explicit ``data=None`` distinguishable from a requests-style + response. + """ + if hasattr(response, "status"): + status = response.status + elif hasattr(response, "status_code"): + status = response.status_code + else: + status = None headers = response.headers if status == 200: content_type = _get_header(headers, "Content-Type") or "" ct_lower = content_type.lower() - body = response.data if response.data is not None else b"" + if hasattr(response, "data"): + body = response.data + elif hasattr(response, "content"): + body = response.content + else: + body = None + if body is None: + body = b"" if ct_lower.startswith(_JSON_CT_PREFIX): try: diff --git a/pipenv/resolver/pure_python_metadata.py b/pipenv/resolver/pure_python_metadata.py new file mode 100644 index 0000000000..b34c82eab4 --- /dev/null +++ b/pipenv/resolver/pure_python_metadata.py @@ -0,0 +1,924 @@ +"""Wheel ``METADATA`` fetcher for the pure-Python resolver backend +(Initiative G Phase 3, T2). + +Two-tier strategy per design §5.2: + +1. **PEP 658 fast path** — when the index advertises ``core-metadata`` + on a candidate, GET ``.metadata`` (or the explicit + advertised URL), verify the advertised hash, and parse via stdlib + :mod:`email.parser`. Cost: one small HTTP GET per wheel (~3–5 kB + typical). + +2. **Wheel-head fallback** — for indexes that don't advertise PEP 658 + or for candidates whose ``.metadata`` file is missing, HTTP ``HEAD`` + the wheel URL to read ``Content-Length``, then issue a range GET + for the last ~64 kB to capture the zip central directory. Parse + the directory via :mod:`zipfile`, locate ``/METADATA``, + then issue a second range covering just that entry, decompress, and + parse. + + When the server rejects ``HEAD`` (some private indexes return 405 / + 403) the fetcher probes with a tiny range ``GET`` and reads the + length out of ``Content-Range`` instead. + +A small on-disk :class:`MetadataCache` keyed by ``sha256(wheel_url)`` +short-circuits both paths on repeat resolves. Wheels are immutable on +PyPI so cache entries are valid forever; corruption is silently +treated as a miss (caller refetches and overwrites — same contract as +:class:`pipenv.resolver.manifest_cache.ParsedManifestCache`). + +Critical constraint (enforced by the T17 pre-commit gate): +**this module must not import from patched-pip's internal package.** +Vendored ``packaging`` and patched-pip's vendored ``urllib3`` / +``requests`` are permitted; ``pip._internal`` is not. The HTTP layer +is a duck-typed ``session`` parameter — production hands us the same +session shape as the PEP 691 client uses; tests pass a +:class:`unittest.mock.MagicMock`. + +The :class:`Candidate` shape (Phase 1) does not yet carry PEP 658 +advertisement. T2 therefore accepts ``metadata_url`` and +``metadata_hash`` as keyword parameters to :func:`fetch_metadata`; +T3 / T7 are free to widen :class:`Candidate` later and pass the values +through transparently. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import logging +import os +import tempfile +import zipfile +from dataclasses import dataclass +from email import policy as email_policy +from email.parser import HeaderParser +from pathlib import Path +from typing import Any + +from pipenv.resolver.candidate import Candidate +from pipenv.vendor.packaging.utils import canonicalize_name + +__all__ = [ + "CoreMetadata", + "MetadataCache", + "MetadataFetchError", + "fetch_metadata", +] + +_LOGGER = logging.getLogger(__name__) + +# How many bytes from the wheel's tail we ask for on the first range +# GET when probing the central directory. 64 kB covers the directory +# of every PyPI wheel surveyed in design §5.2; if the central +# directory is larger, :class:`_PartialFile` re-issues with a bigger +# window. +_CENTRAL_DIRECTORY_PROBE_BYTES = 65_536 + +# Cap on the wheel-tail window before we give up and treat the wheel +# as unsupported by the range-fetch path. 1 MB is well past any real +# central-directory size; beyond this we fall through to a +# :class:`MetadataFetchError` so the backend can surface the issue +# instead of looping on ever-larger ranges. +_MAX_CENTRAL_DIRECTORY_WINDOW = 1_048_576 + +# Timeout budget for one HTTP call. Mirrors the simple-API client's +# defaults (see pep691.py); metadata fetches are part of resolve and +# must fail fast on a flaky mirror. +_DEFAULT_CONNECT_TIMEOUT = 10.0 +_DEFAULT_READ_TIMEOUT = 30.0 + + +# --------------------------------------------------------------------------- +# Public dataclass surface +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class CoreMetadata: + """Parsed core metadata for one wheel candidate. + + Fields mirror PEP 643 + PEP 658's METADATA subset that drives + dependency resolution. All fields except ``name`` and ``version`` + are best-effort: a wheel with a malformed Requires-Python header, + for example, populates the rest of the dataclass and leaves + ``requires_python = None`` so the resolver can still make progress. + """ + + name: str + """PEP 503-canonical package name (lowercase, ``-`` separators).""" + + version: str + """PEP 440 version string as it appears in METADATA.""" + + requires_python: str | None + """Raw ``Requires-Python`` specifier, or ``None`` if absent.""" + + requires_dist: tuple[str, ...] + """Each ``Requires-Dist`` header as a raw string, in source order.""" + + provides_extras: frozenset[str] + """Set of ``Provides-Extra`` names declared by the wheel.""" + + summary: str | None + """One-line ``Summary`` header value, for diagnostics.""" + + +class MetadataFetchError(Exception): + """Raised when metadata cannot be fetched / verified. + + Carries a human-readable message; callers (T9 backend) translate + into ``ResolverResponse.result = InternalError(...)`` so the user + sees the wheel URL and the failure mode. + """ + + +# --------------------------------------------------------------------------- +# On-disk cache +# --------------------------------------------------------------------------- + + +_CACHE_SCHEMA_VERSION = 1 + + +class MetadataCache: + """Filesystem-backed cache of parsed :class:`CoreMetadata`. + + Layout:: + + /.json + + The wheel URL is the cache key because wheels are content-addressed + on PyPI: same URL → same bytes → same metadata, forever. Atomic + writes go through ``tempfile`` + ``os.replace`` exactly the same + way :class:`pipenv.resolver.manifest_cache.ParsedManifestCache` + does, so a crashed write never leaves a partial file at the target. + + Corruption (missing keys, wrong schema_version, unreadable JSON) + is silently treated as a miss; the caller refetches and overwrites + on the next ``put``. + """ + + def __init__(self, root: Path) -> None: + self._root = Path(root) + + def get(self, wheel_url: str) -> CoreMetadata | None: + """Return cached metadata for ``wheel_url`` or ``None`` on miss. + + ``None`` is returned for any failure mode (file missing, + ``OSError`` on read, malformed JSON, wrong schema version, + missing required keys). We never raise on a cache miss — a + corrupt entry must not block a resolve, only force a refetch. + """ + target = self._path_for(wheel_url) + try: + raw = target.read_bytes() + except FileNotFoundError: + return None + except OSError: + return None + + try: + payload = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError): + return None + if not isinstance(payload, dict): + return None + if payload.get("schema_version") != _CACHE_SCHEMA_VERSION: + return None + + try: + return CoreMetadata( + name=payload["name"], + version=payload["version"], + requires_python=payload.get("requires_python"), + requires_dist=tuple(payload.get("requires_dist") or ()), + provides_extras=frozenset(payload.get("provides_extras") or ()), + summary=payload.get("summary"), + ) + except (KeyError, TypeError): + return None + + def put(self, wheel_url: str, metadata: CoreMetadata) -> None: + """Atomically write ``metadata`` to disk under ``wheel_url``'s key. + + Same temp-file + ``os.replace`` pattern as + :class:`ParsedManifestCache.put`: a crash mid-write leaves the + previous payload (if any) intact; the only possible litter is + a ``..tmp`` file that the next successful write + will leave alone. + """ + target = self._path_for(wheel_url) + parent = target.parent + parent.mkdir(parents=True, exist_ok=True) + + payload = { + "schema_version": _CACHE_SCHEMA_VERSION, + "name": metadata.name, + "version": metadata.version, + "requires_python": metadata.requires_python, + "requires_dist": list(metadata.requires_dist), + # Sort so repeated put() of the same metadata produces + # bit-identical files (useful for cache-corruption + # debugging — diffing two cache files highlights the real + # delta, not set ordering). + "provides_extras": sorted(metadata.provides_extras), + "summary": metadata.summary, + } + + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + tmp = tempfile.NamedTemporaryFile( + mode="wb", + delete=False, + dir=str(parent), + prefix=target.name + ".", + suffix=".tmp", + ) + try: + try: + tmp.write(body) + tmp.flush() + os.fsync(tmp.fileno()) + finally: + tmp.close() + os.replace(tmp.name, str(target)) + except BaseException: + try: + os.unlink(tmp.name) + except OSError: + pass + raise + + def _path_for(self, wheel_url: str) -> Path: + """Map ``wheel_url`` → on-disk filename. + + Full sha256 (64 chars) to avoid prefix-collision between two + distinct wheel URLs that happen to share a 64-char prefix — + same rationale as :meth:`ParsedManifestCache._path_for`. + """ + digest = hashlib.sha256(wheel_url.encode("utf-8")).hexdigest() + return self._root / f"{digest}.json" + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def fetch_metadata( + candidate: Candidate, + session: Any, + *, + cache: MetadataCache | None = None, + metadata_url: str | None = None, + metadata_hash: dict[str, str] | None = None, +) -> CoreMetadata: + """Return parsed :class:`CoreMetadata` for ``candidate``. + + Algorithm: + + 1. If ``cache`` is supplied and has an entry for ``candidate.url``, + return it (no network). + 2. If ``metadata_url`` is supplied (PEP 658 fast path), GET it, + verify against ``metadata_hash`` if non-empty, parse. + 3. Otherwise: HEAD the wheel URL to read ``Content-Length`` (fall + back to a tiny range GET if HEAD is rejected), then range-GET + the last 64 kB of the wheel to capture the zip central + directory, locate the ``METADATA`` entry, range-GET its + compressed bytes, decompress, parse. + 4. Populate the cache (if supplied) before returning. + + Raises :class:`MetadataFetchError` on any unrecoverable failure — + hash mismatch, malformed METADATA body, missing METADATA in the + zip, an HTTP-level failure that the simpler probe + range fetch + could not work around, etc. + + Parameters + ---------- + candidate: + The :class:`Candidate` whose metadata we need. + session: + Duck-typed HTTP session whose ``request(method, url, *, + headers=...)`` returns an object with ``.status``, ``.data``, + ``.headers``. Same shape the rest of ``pipenv.resolver`` uses + — production hands us a configured ``urllib3.PoolManager``; + tests use :class:`unittest.mock.MagicMock`. + cache: + Optional :class:`MetadataCache` for read-through caching. + metadata_url: + Optional explicit URL for the PEP 658 ``.metadata`` companion + file. When ``None`` and ``metadata_hash`` is also ``None``, + the wheel-head fallback is used directly. + metadata_hash: + Optional ``{algo: hexvalue}`` mapping advertised by the index + for the metadata body. Only ``sha256`` is honoured; other + algos are ignored. When the dict is empty or absent on the + PEP 658 path, the hash check is skipped (some indexes + advertise the URL but not the hash). + + Sdist branch (Initiative G Phase 3b, T_S2) + ----------------------------------------- + When ``candidate.is_wheel`` is ``False`` we delegate entirely to + :func:`pipenv.resolver.pure_python_sdist.extract_metadata_from_sdist`, + which honours / populates the same :class:`MetadataCache`. The + branch sits **after** the local cache short-circuit so a populated + cache entry skips both the sdist build and the import of the + heavier ``pyproject_hooks`` + ``tarfile`` machinery. The + consequence is a one-extra ``cache.get`` on a cold sdist (T_S1 + will call ``cache.get`` again internally), but that's a cheap + file-stat / read + parse vs. an entire archive download + build — + negligible. The local import in the sdist branch keeps the + wheel-only resolves (the 99 % common case) free of the + ``pyproject_hooks`` + ``tarfile`` + ``zipfile`` import cost. + """ + if cache is not None: + cached = cache.get(candidate.url) + if cached is not None: + return cached + + # T_S2: sdist candidates take a fundamentally different path + # (download + PEP 517 build + parse). Delegate to T_S1's + # extractor, forwarding the same cache instance so the on-disk + # ``sha256(candidate.url)`` key is shared with the wheel side. + if not getattr(candidate, "is_wheel", True): + # Local import: pyproject_hooks + tarfile machinery is heavy + # and not needed for wheel-only resolves. Cold-import cost + # only paid the first time a sdist candidate appears in a + # resolve. + from pipenv.resolver.pure_python_sdist import ( + extract_metadata_from_sdist, + ) + return extract_metadata_from_sdist(candidate, session, cache=cache) + + if metadata_url is not None: + metadata = _fetch_pep658(session, metadata_url, metadata_hash) + else: + metadata = _fetch_via_wheel_head(session, candidate.url) + + if cache is not None: + try: + cache.put(candidate.url, metadata) + except OSError as exc: + # Cache write failure is non-fatal — we have the metadata + # in hand and can return it. Log at debug so cache-disk-full + # scenarios surface in --verbose without poisoning normal runs. + _LOGGER.debug( + "metadata cache write failed for %s: %s", + candidate.url, + exc, + ) + + return metadata + + +# --------------------------------------------------------------------------- +# PEP 658 fast path +# --------------------------------------------------------------------------- + + +def _fetch_pep658( + session: Any, + metadata_url: str, + metadata_hash: dict[str, str] | None, +) -> CoreMetadata: + """GET ``metadata_url``, verify the advertised hash, return parsed body.""" + body = _http_get(session, metadata_url) + _verify_hash(body, metadata_hash, where=metadata_url) + try: + text = body.decode("utf-8") + except UnicodeDecodeError as exc: + # METADATA is RFC 822-ish ASCII / UTF-8 in practice; a non-UTF-8 + # body is almost certainly a captive-portal HTML response or + # similar. Surface it explicitly rather than silently + # producing a half-parsed dataclass. + raise MetadataFetchError( + f"metadata body at {metadata_url} is not UTF-8: {exc}" + ) from exc + return _parse_metadata_text(text) + + +def _verify_hash( + body: bytes, + metadata_hash: dict[str, str] | None, + *, + where: str, +) -> None: + """Verify ``body`` matches ``metadata_hash[sha256]`` if advertised. + + PEP 658 specifies ``sha256`` as the algo in the simple-API + ``core-metadata`` field's dict form. Empty / missing dict skips + the check (some indexes advertise the URL but not the hash; + refusing to fetch them would be worse than trusting the body). + """ + if not metadata_hash: + return + advertised = metadata_hash.get("sha256") + if not advertised: + # No sha256 specifically advertised; nothing we can verify. + return + actual = hashlib.sha256(body).hexdigest() + if actual != advertised: + raise MetadataFetchError( + f"sha256 mismatch on PEP 658 metadata at {where}: " + f"advertised {advertised!r}, computed {actual!r}" + ) + + +# --------------------------------------------------------------------------- +# Wheel-head fallback +# --------------------------------------------------------------------------- + + +def _fetch_via_wheel_head(session: Any, wheel_url: str) -> CoreMetadata: + """Range-fetch the wheel's central directory + METADATA entry, parse. + + Two-step fetch: + + 1. Discover ``Content-Length`` via ``HEAD``. On 405 / 403 / any + non-200 we fall back to a tiny range ``GET`` (``bytes=0-1``) and + read the length from the ``Content-Range`` header. + 2. Range-``GET`` the last ``_CENTRAL_DIRECTORY_PROBE_BYTES`` bytes + — typically 64 kB is enough to cover the entire zip central + directory. If :class:`_PartialFile`'s open call complains the + directory isn't there, widen the window and retry once. + 3. Locate ``/METADATA`` in the directory. Issue a + second range ``GET`` for just that entry's bytes, decompress, + parse. + """ + total_length = _discover_wheel_length(session, wheel_url) + if total_length <= 0: + raise MetadataFetchError( + f"could not determine wheel length for {wheel_url}" + ) + + # Walk the central directory, possibly widening the tail window. + window = min(_CENTRAL_DIRECTORY_PROBE_BYTES, total_length) + while True: + tail_start = max(0, total_length - window) + tail_bytes = _http_get_range( + session, wheel_url, tail_start, total_length - 1 + ) + partial = _PartialFile( + tail_bytes, + offset=tail_start, + total_length=total_length, + session=session, + url=wheel_url, + ) + try: + zf = zipfile.ZipFile(partial) + except zipfile.BadZipFile: + if window >= total_length or window >= _MAX_CENTRAL_DIRECTORY_WINDOW: + raise MetadataFetchError( + f"could not read central directory of {wheel_url} " + f"after sampling {window} bytes" + ) + # Double the window and retry. + window = min(window * 2, total_length, _MAX_CENTRAL_DIRECTORY_WINDOW) + continue + break + + try: + info = _find_metadata_member(zf) + finally: + zf.close() + + if info is None: + raise MetadataFetchError( + f"no /METADATA member found in {wheel_url}" + ) + + # Re-open with the same _PartialFile to extract the METADATA body. + # Re-using ``partial`` is safe — it transparently re-issues range + # GETs for any byte range outside its in-memory window. + partial.seek(0) + zf = zipfile.ZipFile(partial) + try: + with zf.open(info, "r") as fh: + raw = fh.read() + finally: + zf.close() + + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise MetadataFetchError( + f"METADATA body in {wheel_url} is not UTF-8: {exc}" + ) from exc + + return _parse_metadata_text(text) + + +def _discover_wheel_length(session: Any, wheel_url: str) -> int: + """Return the wheel's total byte length via HEAD or a probing GET. + + First tries ``HEAD``: on a 200 with a ``Content-Length`` header we + return it directly. If ``HEAD`` is rejected (405 / 403, some + private indexes don't allow it) or doesn't advertise the length, + we fall through to a tiny range ``GET`` and read the length from + ``Content-Range`` (``bytes 0-1/``). + """ + head_response = _http_request(session, "HEAD", wheel_url) + if head_response is not None: + status = _response_status(head_response) + if status == 200: + length = _content_length(head_response) + if length > 0: + return length + + # Fall back to a probing GET with a tiny range. + probe_response = _http_request( + session, "GET", wheel_url, headers={"Range": "bytes=0-1"} + ) + if probe_response is None: + raise MetadataFetchError( + f"HEAD and probing GET both failed for {wheel_url}" + ) + status = _response_status(probe_response) + if status not in (200, 206): + raise MetadataFetchError( + f"probing GET for {wheel_url} returned HTTP {status}" + ) + content_range = _get_header(probe_response.headers, "Content-Range") + if content_range: + # ``bytes 0-1/``. Split on ``/`` and take the tail. + total_part = content_range.rsplit("/", 1)[-1].strip() + if total_part and total_part != "*": + try: + return int(total_part) + except ValueError: + pass + # Last resort: Content-Length on the probe. This will be 2, not + # the full wheel length, but some misbehaving mirrors set it + # anyway. Better to give up cleanly than spin. + raise MetadataFetchError( + f"could not parse Content-Range from probe GET on {wheel_url}: " + f"{content_range!r}" + ) + + +def _content_length(response: Any) -> int: + """Return ``Content-Length`` from ``response`` or ``-1`` on failure.""" + raw = _get_header(response.headers, "Content-Length") + if not raw: + return -1 + try: + return int(raw) + except (TypeError, ValueError): + return -1 + + +def _find_metadata_member(zf: zipfile.ZipFile) -> zipfile.ZipInfo | None: + """Find the ``-.dist-info/METADATA`` member. + + Wheel layout (PEP 427): exactly one ``*.dist-info/`` directory at + the archive root. The METADATA file lives at + ``/METADATA``. + """ + for info in zf.infolist(): + name = info.filename + if name.endswith(".dist-info/METADATA"): + return info + # Some wheels emit Windows-style separators — accept both. + if name.endswith(".dist-info\\METADATA"): + return info + return None + + +# --------------------------------------------------------------------------- +# Email parser +# --------------------------------------------------------------------------- + + +def _parse_metadata_text(raw: str) -> CoreMetadata: + """Parse a METADATA text body into a :class:`CoreMetadata`. + + Uses :class:`email.parser.HeaderParser` with + :data:`email.policy.compat32` — that's the policy setuptools uses + when *writing* METADATA, so reading with the same policy avoids + folding / re-wrapping surprises on long Requires-Dist headers. + + ``Requires-Dist`` is repeated; we collect every line in source + order via :meth:`Message.get_all`. ``Provides-Extra`` is also + repeated and goes into a :class:`frozenset` (extras are unordered + by spec). + """ + parser = HeaderParser(policy=email_policy.compat32) + msg = parser.parsestr(raw) + + raw_name = msg.get("Name") or "" + raw_version = msg.get("Version") or "" + # canonicalize_name is forgiving on weird casing / separators; + # the rest of the resolver expects PEP 503 canonical names. + name = canonicalize_name(raw_name) if raw_name else "" + version = raw_version.strip() + + requires_python = msg.get("Requires-Python") + if requires_python is not None: + requires_python = requires_python.strip() or None + + requires_dist = tuple( + line.strip() + for line in (msg.get_all("Requires-Dist") or []) + if line and line.strip() + ) + + provides_extras = frozenset( + line.strip() + for line in (msg.get_all("Provides-Extra") or []) + if line and line.strip() + ) + + summary = msg.get("Summary") + if summary is not None: + summary = summary.strip() or None + + return CoreMetadata( + name=name, + version=version, + requires_python=requires_python, + requires_dist=requires_dist, + provides_extras=provides_extras, + summary=summary, + ) + + +# --------------------------------------------------------------------------- +# HTTP helpers +# --------------------------------------------------------------------------- + + +def _http_request( + session: Any, + method: str, + url: str, + *, + headers: dict[str, str] | None = None, +) -> Any: + """Issue a single HTTP request via ``session.request``. + + Returns the response or ``None`` if the session raised — the + fallback path will surface a typed :class:`MetadataFetchError`. + Mirrors :class:`PEP691Client`'s timeout shape so production + behaviour is consistent across resolver HTTP hops. + """ + try: + return session.request( + method, + url, + headers=headers or {}, + timeout=(_DEFAULT_CONNECT_TIMEOUT, _DEFAULT_READ_TIMEOUT), + ) + except Exception as exc: # noqa: BLE001 - logged + swallowed + _LOGGER.debug("HTTP %s failed for %s: %s", method, url, exc) + return None + + +def _response_status(response: Any) -> int: + """Status int from urllib3-style (``.status``) or requests-style + (``.status_code``) responses. Production uses + :class:`PipSession` (requests); the Phase 1+2 unit tests use a + urllib3-style mock — pick by attribute *presence* so explicit + ``None`` values on a urllib3-style mock aren't silently re-interpreted + as a requests response.""" + if hasattr(response, "status"): + status = response.status + elif hasattr(response, "status_code"): + status = response.status_code + else: + status = None + return int(status) if status is not None else 0 + + +def _response_body(response: Any) -> bytes | None: + """Body bytes from urllib3-style (``.data``) or requests-style + (``.content``) responses. Pick by attribute *presence* so an + explicit ``None`` on a urllib3-style mock isn't silently re-read + as ``.content``.""" + if hasattr(response, "data"): + return response.data + if hasattr(response, "content"): + return response.content + return None + + +def _http_get(session: Any, url: str) -> bytes: + """GET ``url`` and return the body bytes, or raise ``MetadataFetchError``.""" + response = _http_request(session, "GET", url) + if response is None: + raise MetadataFetchError(f"GET {url} failed (no response)") + status = _response_status(response) + if status not in (200, 206): + raise MetadataFetchError(f"GET {url} returned HTTP {status}") + data = _response_body(response) + if data is None: + raise MetadataFetchError(f"GET {url} returned no body") + return bytes(data) + + +def _http_get_range( + session: Any, + url: str, + start: int, + end_inclusive: int, +) -> bytes: + """Range-GET ``url`` for bytes ``[start, end_inclusive]``. + + Accepts both 206 (the typical case) and 200 (some mirrors ignore + the ``Range`` header but still return the full body — we trim + locally in :class:`_PartialFile`'s consumer). Raises + :class:`MetadataFetchError` on any other status. + """ + headers = {"Range": f"bytes={start}-{end_inclusive}"} + response = _http_request(session, "GET", url, headers=headers) + if response is None: + raise MetadataFetchError(f"range GET {url} failed (no response)") + status = _response_status(response) + if status not in (200, 206): + raise MetadataFetchError( + f"range GET {url} returned HTTP {status}" + ) + data = _response_body(response) + if data is None: + raise MetadataFetchError(f"range GET {url} returned no body") + return bytes(data) + + +def _get_header(headers: Any, name: str) -> str | None: + """Case-insensitively fetch a header value or ``None``. + + Mirrors :func:`pipenv.resolver.pep691._get_header` so tests can + pass plain ``dict`` headers and production-shaped + ``HTTPHeaderDict`` interchangeably. + """ + if headers is None: + return None + try: + value = headers.get(name) + except Exception: # noqa: BLE001 + value = None + if value is not None: + return value + try: + items = headers.items() + except Exception: # noqa: BLE001 + return None + name_lower = name.lower() + for key, val in items: + if isinstance(key, str) and key.lower() == name_lower: + return val + return None + + +# --------------------------------------------------------------------------- +# Partial-zipfile adapter +# --------------------------------------------------------------------------- + + +class _PartialFile(io.RawIOBase): + """File-like view over a wheel-tail buffer that re-fetches on demand. + + :class:`zipfile.ZipFile` does a single seek-to-the-end-and-walk + pass to discover the central directory, then random-reads each + member's local file header + compressed body. When we hand it a + buffer covering just the last 64 kB of the wheel, the central- + directory walk succeeds locally but a subsequent ``zf.open(...)`` + on a member earlier in the archive needs bytes we don't have. + + This shim implements just enough of the file interface that + :class:`ZipFile` is happy: + + * ``seekable() / readable()`` → ``True``. + * ``seek(offset, whence)`` → tracks position; supports SEEK_SET, + SEEK_CUR, SEEK_END. + * ``tell()`` → current absolute offset within the wheel. + * ``read(size)`` → returns ``size`` bytes, re-fetching via a range + GET if the request falls outside the in-memory window. + + The in-memory window starts as the tail buffer (``offset`` = + ``total_length - len(buffer)`` to ``total_length - 1``). When the + consumer reads outside that range, we issue a range GET for the + missing bytes — expanding the window so subsequent reads in the + same neighbourhood don't re-fetch. + + Memory bound: in the worst case the window grows to the full + wheel length, which matches a direct GET of the wheel and is the + fundamental cost ceiling. In practice a single ``METADATA`` read + expands the window by at most ~ a few kB beyond the directory. + """ + + def __init__( + self, + tail_buffer: bytes, + *, + offset: int, + total_length: int, + session: Any, + url: str, + ) -> None: + super().__init__() + # Buffer holds bytes ``[buffer_start, buffer_end)`` of the wheel. + self._buffer = bytearray(tail_buffer) + self._buffer_start = offset + self._total_length = total_length + self._session = session + self._url = url + self._position = 0 + + # ---- io.RawIOBase contract --------------------------------------- + + def seekable(self) -> bool: # noqa: D401 - one-liner + return True + + def readable(self) -> bool: # noqa: D401 - one-liner + return True + + def writable(self) -> bool: # noqa: D401 - one-liner + return False + + def tell(self) -> int: # noqa: D401 - one-liner + return self._position + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + if whence == io.SEEK_SET: + new = offset + elif whence == io.SEEK_CUR: + new = self._position + offset + elif whence == io.SEEK_END: + new = self._total_length + offset + else: + raise ValueError(f"unsupported whence: {whence}") + if new < 0: + raise ValueError(f"negative seek position {new}") + self._position = new + return new + + def read(self, size: int = -1) -> bytes: + if size is None or size < 0: + size = self._total_length - self._position + if size <= 0 or self._position >= self._total_length: + return b"" + end = min(self._position + size, self._total_length) + self._ensure_range(self._position, end) + start_in_buf = self._position - self._buffer_start + end_in_buf = end - self._buffer_start + data = bytes(self._buffer[start_in_buf:end_in_buf]) + self._position = end + return data + + # ---- internals --------------------------------------------------- + + def _buffer_end(self) -> int: + """Exclusive end offset of the in-memory window.""" + return self._buffer_start + len(self._buffer) + + def _ensure_range(self, start: int, end: int) -> None: + """Ensure bytes ``[start, end)`` are covered by the buffer. + + Grows the buffer at the low end and/or the high end as + necessary by issuing range GETs. No-op when the buffer + already covers the request. + """ + if start < self._buffer_start: + # Need a prefix fetch. Pull ``[start, buffer_start)``. + prefix = _http_get_range( + self._session, + self._url, + start, + self._buffer_start - 1, + ) + # If the server ignored Range and returned the whole file, + # ``prefix`` may be longer than the requested span. Trim. + wanted = self._buffer_start - start + if len(prefix) > wanted: + prefix = prefix[:wanted] + elif len(prefix) < wanted: + # Short read on a range GET is rare but possible on a + # mirror that capped the response. Surface it. + raise MetadataFetchError( + f"short range read on {self._url}: " + f"asked {wanted} bytes got {len(prefix)}" + ) + self._buffer = bytearray(prefix) + self._buffer + self._buffer_start = start + if end > self._buffer_end(): + # Need a suffix fetch. Pull + # ``[buffer_end, end)`` (inclusive end_inclusive = end-1). + current_end = self._buffer_end() + suffix = _http_get_range( + self._session, + self._url, + current_end, + end - 1, + ) + wanted = end - current_end + if len(suffix) > wanted: + # Trim oversized response from a Range-ignoring mirror. + suffix = suffix[:wanted] + elif len(suffix) < wanted: + raise MetadataFetchError( + f"short range read on {self._url}: " + f"asked {wanted} bytes got {len(suffix)}" + ) + self._buffer.extend(suffix) diff --git a/pipenv/resolver/pure_python_provider.py b/pipenv/resolver/pure_python_provider.py new file mode 100644 index 0000000000..4c69550f0d --- /dev/null +++ b/pipenv/resolver/pure_python_provider.py @@ -0,0 +1,1479 @@ +"""Pure-Python ``resolvelib.AbstractProvider`` adapter for the +in-tree resolver backend (Initiative G phase 3). + +This module is the third leg of the Phase 3 trio: + +* :mod:`pipenv.resolver.pure_python_requirement` — typed + :class:`Requirement` (T1, shipped at commit ``a5ac1eff``). +* :mod:`pipenv.resolver.pure_python_metadata` — :class:`MetadataFetcher` + (T2, shipped at commit ``93bc3f74``). +* **This module** — :class:`PurePythonProvider` that consumes both and + drives ``resolvelib.Resolver`` directly, replacing pip's + ``PipProvider`` / ``LinkEvaluator`` / ``Wheel`` pipeline. + +The class is built up across T3–T7: + +* **T3 (this commit)** — :meth:`PurePythonProvider.__init__` + + :meth:`PurePythonProvider.identify`. +* **T4** — :meth:`PurePythonProvider.find_matches`. +* **T5** — :meth:`PurePythonProvider.get_preference`. +* **T6** — :meth:`PurePythonProvider.is_satisfied_by`. +* **T7** — :meth:`PurePythonProvider.get_dependencies`. + +The four un-implemented hot-path methods raise :class:`NotImplementedError` +with the task label (``"T4"`` etc.) so the eventual integration tests +fail loud with a clear pointer at which follow-up task is missing, +rather than silently returning ``None``. + +Critical constraint (enforced by Phase 1's pre-commit grep gate): +**this module must not import from patched-pip's internal package.** +The base class :class:`AbstractProvider` comes from +:mod:`pipenv.patched.pip._vendor.resolvelib.providers` — that path is +``_vendor`` (a third-party-vendored dependency of patched pip), not +``_internal`` (pip's own modules), so the gate is satisfied. + +See ``docs/dev/initiative-g-phase3-design.md`` §5.3 for the design +brief and ``initiative-g-phase3-plan.md`` T3 for the validation +matrix. +""" + +from __future__ import annotations + +from dataclasses import replace as _dataclass_replace +from typing import Any, Iterable, Mapping, Sequence + +from pipenv.patched.pip._vendor.resolvelib.providers import AbstractProvider +from pipenv.resolver.pure_python_requirement import Requirement +from pipenv.vendor.packaging.markers import Marker +from pipenv.vendor.packaging.requirements import Requirement as PackagingRequirement +from pipenv.vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet +from pipenv.vendor.packaging.utils import canonicalize_name +from pipenv.vendor.packaging.version import InvalidVersion, Version + +__all__ = ["PurePythonProvider", "_SdistEncountered", "_drive_resolver"] + + +def _strip_extra_clauses(marker: Marker) -> Marker | None: + """Return a copy of ``marker`` with every ``extra == X`` / + ``extra != X`` clause removed. + + Used by T7's :meth:`PurePythonProvider.get_dependencies` to scrub + the extras-gating clauses out of a transitive's marker once the + parent's extras context has already accepted the requirement. + The remaining clauses (e.g. ``python_version >= '3.11'``, + ``sys_platform == 'linux'``) stay so the runtime + :meth:`is_satisfied_by` check still enforces them. + + Returns ``None`` when nothing of substance remains — e.g. the + input was ``extra == "foo"`` alone, or every clause referenced + ``extra``. ``None`` matches the existing + :attr:`Requirement.marker` shape for the "no runtime marker" + case. + + Walks :attr:`Marker._markers` — a recursive + ``list[tuple | str | list]`` structure where tuples are + ``(Variable, Op, Value)`` comparisons, ``str`` slots are the + boolean joiners ``"and"`` / ``"or"``, and nested lists are + parenthesised sub-expressions. After filtering, dangling + joiners (a leading ``"and"`` or trailing ``"or"``) are dropped + so the re-serialised marker string parses cleanly. + """ + def _is_extra_clause(node: Any) -> bool: + if not isinstance(node, tuple) or len(node) != 3: + return False + lhs, _op, rhs = node + lhs_name = getattr(lhs, "value", None) + rhs_name = getattr(rhs, "value", None) + return lhs_name == "extra" or rhs_name == "extra" + + def _walk(markers: list[Any]) -> list[Any]: + out: list[Any] = [] + for node in markers: + if isinstance(node, list): + sub = _walk(node) + if sub: + out.append(sub) + elif isinstance(node, tuple): + if _is_extra_clause(node): + continue + out.append(node) + else: + # Boolean joiner string ("and" / "or") — keep for now; + # tidy up dangling joiners below. + out.append(node) + # Drop a leading / trailing joiner left behind by a stripped + # clause at the edge of the list. + while out and isinstance(out[0], str): + out.pop(0) + while out and isinstance(out[-1], str): + out.pop() + # Collapse runs of consecutive joiner strings (can happen when + # both sides of an ``and``/``or`` were stripped) — keep the + # first. This is rare; most stripped clauses sit at one end. + cleaned: list[Any] = [] + prev_joiner = False + for node in out: + is_joiner = isinstance(node, str) + if is_joiner and prev_joiner: + continue + cleaned.append(node) + prev_joiner = is_joiner + return cleaned + + stripped = _walk(marker._markers) + if not stripped: + return None + # Re-serialise and re-parse so the returned object is a fresh + # :class:`Marker` whose internal state is consistent with its + # string form (consumers like the lockfile emitter rely on + # ``str(marker)`` matching the parsed structure). + from pipenv.vendor.packaging.markers import _format_marker + + try: + return Marker(_format_marker(stripped)) + except Exception: # noqa: BLE001 + # Defensive: a malformed re-serialisation falls back to None + # (no runtime marker) — pip's stance on unparseable markers + # is the same. + return None + + +class _SdistEncountered(Exception): + """DEPRECATED (Phase 3b T_S3). Phase 3a's fail-loud sdist signal. + + Phase 3b builds sdists transparently via T_S1 (PEP 517 METADATA + extraction in :mod:`pipenv.resolver.pure_python_sdist`) and T_S2 + (the routing branch in :meth:`MetadataFetcher.fetch_metadata`), + so this exception is no longer raised from production code. + + The class is preserved as an importable symbol for back-compat + with external test suites that referenced the Phase 3a contract; + the constructor still accepts a candidate and records it on + :attr:`candidate` so a legacy caller that constructs (but does + not re-raise) the exception keeps working. + + History (for the audit trail): in Phase 3a, per the design Q-A + sign-off 2026-05-12, :meth:`PurePythonProvider.get_dependencies` + raised this when handed a non-wheel candidate and T9's + :class:`PurePythonBackend.resolve` translated it into a + structured :class:`InternalError` response. The Phase 3b T_S3 + work (commit landing this docstring) removed both the raise site + and the corresponding handler. + """ + + def __init__(self, candidate: Any) -> None: + # Stored as attribute so T9's catch site can read it back + # without parsing the exception message. + self.candidate = candidate + # Message shape mirrors the design §7 Q-A example: name + + # version is the minimum context a user needs to find the + # offending Pipfile entry / transitive parent chain. + name = getattr(candidate, "name", "") + version = getattr(candidate, "version", "") + super().__init__( + f"sdist-only candidate {name} {version!r}" + ) + + +# Identifier shape — matches the design §5.3 contract: a 2-tuple of +# ``(canonical_name, frozenset(extras))``. Pinned here as a documented +# alias so T4 / T5 (which consume the identifier from +# ``resolvelib.Resolver`` callbacks) reference the same shape without +# re-declaring it. +Identifier = tuple[str, frozenset[str]] + + +class PurePythonProvider(AbstractProvider): + """``resolvelib.AbstractProvider`` over :class:`Requirement` + + :class:`pipenv.resolver.candidate.Candidate`. + + Phase 3 design §5.3. The constructor takes the four collaborators + the hot-path methods (T4–T7) need: + + * ``cache`` — :class:`pipenv.resolver.manifest_cache.ParsedManifestCache` + (or any object exposing the same ``.get(index_url, name)`` shape). + T4's :meth:`find_matches` reads candidate lists from here. + * ``fetcher`` — :class:`pipenv.resolver.fetcher.ParallelFetcher`. + T4 calls ``.populate([(index_url, name)])`` on cache miss. + * ``metadata_fetcher`` — :class:`pipenv.resolver.pure_python_metadata.MetadataFetcher`. + T7's :meth:`get_dependencies` invokes ``.fetch_metadata(candidate)`` + to retrieve ``Requires-Dist`` for wheel candidates. + * ``target_env`` — a mapping of marker-variable → value (e.g. + ``{"python_version": "3.12", "sys_platform": "linux"}``). T6 + + T7 use this to evaluate :class:`Marker` objects. + + All four are stored verbatim; this class does not own their + lifecycle. Tests pass plain sentinels for the cache / fetcher / + metadata_fetcher when only T3's behaviour is under test. + + Why kwargs-only + --------------- + The ``__init__`` signature is ``*, cache, fetcher, metadata_fetcher, + target_env`` — keyword-only — so that adding a new collaborator in + Phase 4 (e.g. an ``index_url`` resolver) doesn't break the call + sites in T8's smoke test and T9's backend. All four call sites + pass the args by name today. + """ + + def __init__( + self, + *, + cache: Any, + fetcher: Any, + metadata_fetcher: Any, + target_env: Any, + index_urls: Sequence[str] = ("https://pypi.org/simple",), + allow_prereleases: bool = False, + ) -> None: + # ``Any`` rather than concrete types so unit tests can pass + # plain object sentinels for the fields T3 doesn't touch. T4–T7 + # tighten types as they consume each collaborator — at which + # point a static checker will catch a wrong shape at that call + # site, not here. + self._cache = cache + self._fetcher = fetcher + self._metadata_fetcher = metadata_fetcher + self._target_env = target_env + # T4: ``index_urls`` is the list of simple-API index URLs to + # consult for candidates. Stored as a tuple so the provider + # can't mutate it mid-resolve (frozen-by-convention). Order + # matters — earlier indexes win on duplicate ``(name, version, + # filename)`` keys (mirrors pip's --index-url precedence). + self._index_urls: tuple[str, ...] = tuple(index_urls) + # T4: ``allow_prereleases`` flips the default + # ``SpecifierSet.contains(prereleases=...)`` flag. When + # ``False`` (default), prereleases are admitted only when the + # specifier itself opts in (e.g. ``>=2.0a1``) — same semantics + # as pip's ``--pre``. + self._allow_prereleases: bool = bool(allow_prereleases) + # T_PARITY_REAL extras-work follow-up (2026-05-13): track + # conflict-promotion state across :meth:`narrow_requirement_selection` + # calls. Mirrors pip's ``_conflict_counts`` / + # ``_conflict_promoted`` pair in + # ``pipenv/patched/pip/_internal/resolution/resolvelib/provider.py``. + # Without these, ``get_preference``'s ``backtrack_causes``-only + # signal is too local — by the time a high-conflict identifier + # gets enough backtrack causes to register, the resolver has + # already pinned a wide unconflicted layer that has to be + # rolled back. ``narrow_requirement_selection`` provides the + # eager promote-on-first-conflict path. + from collections import defaultdict + + self._conflict_counts: dict[Identifier, int] = defaultdict(int) + self._conflict_promoted: set[Identifier] = set() + + # ------------------------------------------------------------------ + # T3 — identify + # ------------------------------------------------------------------ + + def identify(self, requirement_or_candidate: Any) -> Identifier: + """Group-key for ``resolvelib``: ``(canonical_name, frozenset(extras))``. + + Accepts both :class:`Requirement` and + :class:`pipenv.resolver.candidate.Candidate` instances. Two + graph elements with the same identifier are merged into the + same group by ``resolvelib`` — different extras are different + groups by design (e.g. ``django`` vs ``django[argon2]``). + + Implementation note + ------------------- + We branch on ``isinstance(req_or_cand, Requirement)`` for the + :class:`Requirement` path so static checkers see the type + narrowing on ``.extras``. The :class:`Candidate` path uses + ``getattr(..., "extras", frozenset())`` — duck-typing over + ``.name`` and ``.extras`` rather than importing ``Candidate`` + directly. That keeps this module's import graph one node + smaller and dodges any future circular if T9's backend wants + to hand the provider a :class:`Candidate` subclass. + + Both branches return a 2-tuple whose second element is a + :class:`frozenset` — never ``None``, never a ``list`` — so + ``hash(identifier)`` works unconditionally and the + ``resolvelib`` invariant "identifiers are hashable" holds. + + The ``name`` field is already PEP-503-canonical on + :class:`Requirement` (T1's ``from_pipfile_entry`` runs + ``canonicalize_name``). :class:`Candidate.name` is documented + as canonical too (caller's responsibility, see the + ``Candidate`` field docstring) — we don't re-canonicalise. + """ + if isinstance(requirement_or_candidate, Requirement): + return ( + requirement_or_candidate.name, + requirement_or_candidate.extras, + ) + + # Candidate (or any duck-type with ``.name`` / ``.extras``). + # ``.extras`` defaults to ``frozenset()`` if missing — survives + # against pre-T3 ``Candidate`` instances that pre-date the + # field addition (defensive: ``Candidate`` itself defaults to + # ``frozenset()`` via ``field(default_factory=frozenset)`` so + # this fallback should never fire in practice). + name: str = requirement_or_candidate.name + extras: frozenset[str] = getattr( + requirement_or_candidate, "extras", frozenset() + ) + return (name, extras) + + # ------------------------------------------------------------------ + # T4 — find_matches + # ------------------------------------------------------------------ + + def find_matches( + self, + identifier: Identifier, + requirements: Mapping[Identifier, Iterable[Requirement]], + incompatibilities: Mapping[Identifier, Iterable[Any]], + ) -> list[Any]: + """Return compatible :class:`Candidate`\\ s ordered + highest-version-first. + + Owned by T4 — the hot path of resolution. Reads candidate lists + from :class:`pipenv.resolver.manifest_cache.ParsedManifestCache` + for each configured index URL, filters them against the merged + constraints carried in ``requirements`` (specifier intersection) + and the rejected-set in ``incompatibilities``, applies the + candidate's own ``requires_python`` gate against + ``self._target_env["python_version"]``, and returns the + filtered list sorted by :class:`packaging.version.Version` + descending. + + Algorithm + --------- + 1. Unpack ``(name, _extras)`` from ``identifier``. + 2. Materialise ``requirements[identifier]`` (an iterator — + consume eagerly because resolvelib hands us a one-shot view). + 3. Pull cached candidates for ``(index_url, name)`` from every + configured index. Concatenate non-``None`` results. + 4. If every index missed, dispatch a single + :meth:`fetcher.populate` for the package — single round-trip + per index — and re-read the cache. Still missing on every + index → return ``[]`` (no candidates available; caller will + treat as a resolution dead-end rather than this method + raising). + 5. Deduplicate by ``(name, version, filename)``. Same artifact + served by mirror indexes shouldn't be doubled. + 6. Build the rejected-version-by-identity tuple set for + ``incompatibilities[identifier]``. + 7. Filter: + * every ``req.specifier.contains(version, prereleases=...)`` + holds across all ``reqs``; + * the candidate's identity tuple is not in ``rejected``; + * the candidate's ``requires_python`` (if non-empty) admits + ``target_env["python_version"]``. + 8. Sort by :class:`packaging.version.Version` descending. + + sdist / wheel handling + ---------------------- + Per the Phase 3 design (Q-A) ``find_matches`` returns sdists + AND wheels — the sdist-only signal triggers later inside + :meth:`get_dependencies` (T7) when the resolver actually tries + to expand a transitive's deps. Q-F's top-level pre-check + (T9) is a separate concern and runs in + :class:`PurePythonBackend.resolve` before this method is + invoked. Returning all artifacts here keeps the layering + clean. + + Cache-key caveat + ---------------- + The :class:`ParsedManifestCache` is keyed on + ``(index_url, package_name)`` — the provider doesn't know which + index served a given cached candidate, so we iterate every + configured ``index_url`` and concatenate hits. Mirrors pip's + ``PackageFinder``'s multi-index walk. T5/T6 follow-up: if the + index identity ever becomes load-bearing for an ordering + decision, widen ``Candidate`` with an ``index_url`` field + rather than threading it through here. + """ + name, _extras = identifier + reqs: list[Requirement] = list(requirements.get(identifier, [])) + + # Step 3-4: read cache; on full miss, dispatch populate then + # re-read. We use a tiny private helper because the same shape + # runs twice. + cached = self._collect_cached_candidates(name) + if not cached: + populate = getattr(self._fetcher, "populate", None) + if populate is not None: + populate([(idx, name) for idx in self._index_urls]) + cached = self._collect_cached_candidates(name) + if not cached: + return [] + + # Step 5: dedup on ``(name, version, filename)`` — same wheel + # served by mirror indexes is a single artifact. + seen: set[tuple[str, str, str]] = set() + unique: list[Any] = [] + for cand in cached: + key = (cand.name, cand.version, cand.filename) + if key in seen: + continue + seen.add(key) + unique.append(cand) + + # Step 5b: per-version sdist suppression. When a release has a + # wheel anywhere in the cache (even if that wheel later gets + # rejected by ``rejected`` / requires_python / yanked filters), + # the sdist is not a useful alternative — it's the same package, + # same metadata, and would produce the same downstream conflict + # at the cost of a PEP 517 build. Pip's ``PackageFinder`` + # publishes one canonical Link per version (wheel preferred); + # we mirror that here by dropping sdists whose version has a + # wheel companion BEFORE the reject/satisfies filtering, so + # backtracking past the wheel doesn't rediscover the sdist. + # Bench fixture trigger: ``sentry-protos`` / ``grpcio-status`` + # exploding into dozens of sdist builds during backtracking. + wheel_versions_in_cache = { + c.version for c in unique if c.is_wheel + } + unique = [ + c for c in unique + if c.is_wheel or c.version not in wheel_versions_in_cache + ] + + # Step 6: rejected set, keyed on ``(name, version, filename)`` + # so an incompatibility from a different cache copy still + # matches by identity tuple. + rejected: set[tuple[str, str, str]] = { + (c.name, c.version, c.filename) + for c in incompatibilities.get(identifier, []) + } + + # Step 7: filter. Two-pass prerelease policy mirrors pip's + # ``get_applicable_candidates`` + # (``pipenv/patched/pip/_internal/index/package_finder.py``): + # the strict pass rejects every prerelease unconditionally; if + # it yields zero candidates AND ``_allow_prereleases`` isn't + # set, the PEP-440 fallback pass re-runs with prereleases + # admitted. See :meth:`_candidate_satisfies_requirements` for + # the per-spec ``prereleases`` handling and the rationale for + # not using the broken ``prereleases=None`` per-item shape. + target_python = None + if isinstance(self._target_env, Mapping): + target_python = self._target_env.get("python_version") + + def _filter_pass(strict_prereleases: bool) -> list[Any]: + out: list[Any] = [] + for cand in unique: + cand_key = (cand.name, cand.version, cand.filename) + if cand_key in rejected: + continue + if not self._candidate_satisfies_requirements( + cand, reqs, strict_prereleases=strict_prereleases + ): + continue + if not self._candidate_requires_python_ok(cand, target_python): + continue + if self._candidate_is_skippable_yanked(cand, reqs): + continue + out.append(cand) + return out + + filtered = _filter_pass(strict_prereleases=True) + if not filtered and not self._allow_prereleases: + # PEP-440 fallback: no stables matched, admit prereleases. + filtered = _filter_pass(strict_prereleases=False) + + # Step 8: sort by ``(Version, is_wheel)`` descending. Highest + # version wins; within a version, wheel beats sdist + # (``True > False`` and ``reverse=True`` lifts wheels above + # sdists at the same version). Mirrors pip's + # ``PackageFinder._sort_links_by_priority`` behaviour: when an + # artifact exists as both a wheel and an sdist at the same + # release, the wheel is the canonical pick — picking the sdist + # forces a PEP 517 build for nothing and (worse) can break + # resolution when the sdist's build backend isn't installable + # (see ``python3-saml 1.16.0`` / ``redis-py-cluster 2.1.3`` on + # the bench fixture, both wheel-available but pure-python was + # silently routing to the sdist build path). + # + # ``InvalidVersion`` is surfaced as a parse error rather than + # silently sorting last — an unparseable version in the cache + # is a bug upstream (PEP 691 parser should have rejected it) + # and we want loud failure here. + filtered.sort( + key=lambda c: (Version(c.version), bool(c.is_wheel)), + reverse=True, + ) + + # Step 9: propagate the identifier's extras onto each candidate. + # The PEP 691 simple-API doesn't report ``provides_extras``, so + # the cache's :class:`Candidate.extras` always parses as + # ``frozenset()``. Pip's :class:`ExtrasCandidate` wraps a base + # candidate with the requested extras so that + # :meth:`get_dependencies` can iterate marker-gated + # ``Requires-Dist`` entries against the right + # ``extra=`` context (see + # ``pipenv/patched/pip/_internal/resolution/resolvelib/candidates.py``). + # Our pure-python analog: when the resolvelib identifier carries + # extras (the user wrote ``pkg = { extras = [...], ... }`` or a + # transitive issued ``pkg[extras]``), clone the filtered + # candidates so ``candidate.extras`` matches the identifier. + # T7's :meth:`get_dependencies` reads ``candidate.extras`` as + # ``parent_extras`` and feeds it into + # :meth:`_marker_active_for_extras`; without this step the + # ``extra == ""`` marker on the transitive line + # (e.g. ``psycopg-binary; extra == "binary"`` on + # ``psycopg[binary]``) evaluates False under ``extra=""`` and + # the transitive silently disappears from the lockfile. + # T_PARITY_REAL bench trigger: project 01 (``django`` + + # ``psycopg[binary]``) — pp's lock was missing + # ``psycopg-binary``. + if _extras: + filtered = [ + _dataclass_replace(cand, extras=_extras) for cand in filtered + ] + return filtered + + # -- T4 helpers ----------------------------------------------------- + + def _collect_cached_candidates(self, name: str) -> list[Any]: + """Walk every configured index URL, concatenating non-empty + cache hits. Returns ``[]`` when every index misses.""" + out: list[Any] = [] + for index_url in self._index_urls: + manifest = self._cache.get(index_url, name) + if manifest is None: + continue + # ``manifest.candidates`` is a tuple per + # :class:`CachedManifest`; extend rather than append. + out.extend(manifest.candidates) + return out + + def _candidate_satisfies_requirements( + self, + candidate: Any, + reqs: Sequence[Requirement], + *, + strict_prereleases: bool = True, + ) -> bool: + """Return ``True`` iff the candidate version satisfies every + ``req.specifier`` in ``reqs``. + + An empty ``reqs`` is treated as "any version is acceptable" — + ``resolvelib`` can legitimately call ``find_matches`` with no + requirements registered for the identifier yet (the identifier + is in the graph but every constraint has already been pruned). + + Prerelease policy: ``_allow_prereleases`` (the ``--pre`` flag + analog) or ``specifier.prereleases is True`` (the specifier + itself opts in, e.g. ``>=2.0a1``) admits prereleases for that + requirement. Otherwise, ``strict_prereleases`` decides: + + * ``True`` (default, the first :meth:`find_matches` pass) — + pass ``prereleases=False`` to :meth:`SpecifierSet.contains`, + which strictly rejects every prerelease. This mirrors pip's + ``get_applicable_candidates`` first pass. + + * ``False`` (the PEP-440 fallback pass) — pass + ``prereleases=True`` so prereleases are admitted when the + strict pass yielded nothing. + + Why we don't use ``prereleases=None`` + ------------------------------------- + ``SpecifierSet.contains(v, prereleases=None)`` is implemented + via :meth:`SpecifierSet.filter` over a SINGLE-element iterable. + The filter then applies PEP-440's "no final release matched, + so accept prereleases" fallback on that one-element list and + admits the prerelease unconditionally. Pip avoids this trap + by calling ``specifier.filter()`` over the FULL candidate list + at once (see + ``pipenv/patched/pip/_internal/index/package_finder.py`` + ``get_applicable_candidates``), where the fallback only + triggers when zero stables matched. Our per-candidate shape + can't replicate that semantics with ``prereleases=None`` — + the two-pass scheme in :meth:`find_matches` is the equivalent. + Bench fixture trigger: ``billiard 4.3.0rc1``, + ``hiredis 3.4.0.dev0``, ``sentry-sdk 3.0.0a7`` all admitted + under transitive ``>=X`` constraints when pip picked the + stable below. + """ + try: + version_obj = Version(candidate.version) + except InvalidVersion: + # Same loud-failure rationale as the sort step — the cache + # should never carry an unparseable version. + return False + for req in reqs: + spec = req.specifier + prereleases: bool + if self._allow_prereleases or spec.prereleases: + prereleases = True + elif strict_prereleases: + prereleases = False + else: + prereleases = True + if not spec.contains(version_obj, prereleases=prereleases): + return False + return True + + def _candidate_requires_python_ok( + self, + candidate: Any, + target_python: str | None, + ) -> bool: + """Return ``True`` iff the candidate's ``requires_python`` + admits ``target_python``. + + ``requires_python`` of ``None`` / empty string is treated as + "no constraint" and accepted. ``target_python`` of ``None`` + (no marker info available) is also treated as accept — the + caller can pin a stricter ``target_env`` if it cares. + + An unparseable ``requires_python`` falls through to accept + rather than silently dropping the candidate — mirrors pip's + :func:`evaluate_link` behaviour (an index with a malformed + ``requires-python`` advertisement shouldn't make the package + invisible). + """ + requires_python = getattr(candidate, "requires_python", None) + if not requires_python: + return True + if not target_python: + return True + try: + spec = SpecifierSet(requires_python) + except InvalidSpecifier: + return True + # ``requires-python`` SpecifierSets typically aren't marked + # ``prereleases=True`` even though Python prereleases (e.g. + # ``3.13.0a1``) routinely appear as ``target_python``. Pass + # ``prereleases=True`` here so an alpha Python isn't silently + # dropped — pip does the same. + try: + return spec.contains(target_python, prereleases=True) + except InvalidVersion: + return True + + def _candidate_is_skippable_yanked( + self, + candidate: Any, + reqs: Sequence[Requirement], + ) -> bool: + """Return ``True`` for a yanked candidate that should be filtered out. + + PEP 592: a yanked release is excluded from automatic selection. + Clients SHOULD only install yanked files when the user explicitly + pins to that exact version — i.e. at least one ``Requirement`` + carries an ``==`` specifier matching the + candidate's version. + + Mirrors pip's behaviour + (:meth:`pip._internal.index.collector.LinkCollector` filters + yanked links unless the requirement is an exact-version pin). + + Returns ``False`` for non-yanked candidates (no skipping needed). + Returns ``False`` for yanked candidates that *are* explicitly + pinned (the user opted in). Returns ``True`` only when a + yanked candidate would be picked unintentionally. + """ + if not getattr(candidate, "yanked", False): + return False + try: + cand_ver = Version(candidate.version) + except InvalidVersion: + # Unparseable version — defer to the version-sort step's + # loud-failure path; don't second-guess here. + return False + for req in reqs: + for spec in req.specifier: + if spec.operator != "==": + continue + try: + if Version(spec.version) == cand_ver: + return False + except InvalidVersion: + continue + return True + + # ------------------------------------------------------------------ + # T5 — get_preference (Q-C strict mirror of pip's tuple shape) + # ------------------------------------------------------------------ + + def get_preference( + self, + identifier: Identifier, + resolutions: Mapping[Identifier, Any], + candidates: Mapping[Identifier, Iterable[Any]], + information: Mapping[Identifier, Iterable[Any]], + backtrack_causes: Sequence[Any], + ) -> tuple: + """Return a sort key driving ``resolvelib``'s ordering — strict + mirror of pip's ``get_preference`` per Initiative G Phase 3 Q-C. + + Source (side-by-side audit reference) + ------------------------------------- + pip's canonical implementation lives at + ``pipenv/patched/pip/_internal/resolution/resolvelib/provider.py`` + in ``PipProvider.get_preference`` (around line 176 — the line + number drifts with pip-vendor updates). The pip tuple in + ``return``-order at the time of writing (low = preferred): + + 1. ``not conflict_promoted`` — identifiers that have repeatedly + caused conflicts get promoted to the front via the separate + ``narrow_requirement_selection`` callback. + 2. ``not direct`` — ``ExplicitRequirement`` (URL-direct) entries. + 3. ``not pinned`` — ``==`` with no wildcard. + 4. ``not upper_bounded`` — ``<``, ``<=``, ``~=``, ``==.*``. + 5. ``requested_order`` — integer rank of the identifier in + ``_user_requested`` (``math.inf`` when not user-requested). + 6. ``not unfree`` — has at least one operator. + 7. ``identifier`` — lexicographic tie-breaker. + + Our mirror — Q-C and the design §5.3 summary + -------------------------------------------- + We render the four leading axes called out in the plan T5 + validation matrix. See the parity-divergence bullet on the + T5 plan entry for the components we deliberately do NOT + mirror today (and why): + + * ``not conflict_promoted`` — derived directly from + ``backtrack_causes`` here, NOT from a separate promoted-set + maintained by ``narrow_requirement_selection``. We don't + ship ``narrow_requirement_selection`` in Phase 3, so the + backtrack count is the closest signal available. + * ``not direct`` — pip's "direct" is ``ExplicitRequirement`` + (URL-direct). Our :class:`Requirement` doesn't model + URL-direct yet (T1's ``source`` is one of + ``{"pipfile", "transitive", "constraint"}``); we render + the closest analog by treating ``source == "pipfile"`` as + Pipfile-direct. Same ordering intent: user-declared beats + transitive. + * ``not upper_bounded`` — kept (matches pip's slot). + * ``requested_order`` — pip's ``_user_requested`` map isn't + available to us; omitted. Lockfile parity is still gated + on T15 / T_PARITY_REAL which will surface any user-visible + divergence. + + Tuple shape this method returns (low = preferred): + + (not has_backtracked, # False (has caused backtracks) first + not is_pipfile, # False (pipfile) first + not is_pinned, # False (pinned) first + not is_upper_bounded, # False (has <,<=,~=,==*) first + not is_unfree, # False (has any op) first + identifier_name) # alphabetical tie-break + + T_PARITY_REAL extras-roundtrip note (2026-05-13) + ------------------------------------------------ + The ``not has_backtracked`` slot used to be a bare + ``backtrack_count`` ascending — i.e. identifiers with zero + backtracks sorted FIRST, identifiers that had caused + conflicts sorted LAST. That's the inverse of pip's intent: + pip's ``not promoted`` (``PipProvider.get_preference``, the + first slot) puts conflict-promoted identifiers FIRST so the + resolver surfaces the conflict early and prunes the + backtrack tree, instead of pinning a wide unconflicted + layer only to roll it all back later. Flipping to a + boolean signal — ``False`` (has backtracked) sorts before + ``True`` (clean record) — matches pip's promote-to-front + intent. Bench-fixture trigger: with the extras-roundtrip + fix in place, the constraint graph picks up + ``protobuf<6`` (from ``sentry-protos``) plus + ``grpcio-status<2,>=1.49.1`` (from + ``google-api-core[grpc]``) plus the rest of the + google-cloud-* ``protobuf<7,>=4.25`` constraints; without + promote-to-front, the resolver thrashes through dozens of + sentry-kafka-schemas / sentry-protos combinations before + backtracking far enough to pick ``sentry-kafka-schemas + 0.1.x`` (which doesn't pull sentry-protos at all). + + See ``docs/dev/initiative-g-phase3-design.md`` §5.3 and + ``initiative-g-phase3-plan.md`` T5 for rationale. + """ + # Step 1: collect the ``RequirementInformation`` rows for this + # identifier. Mirrors pip's ``has_information`` guard at + # ``provider.py`` lines 207-227: information may be absent + # transiently between state transitions and we must not blow + # up on that. + info_rows = list(information.get(identifier, [])) + + # Step 2: source-flag — ``source == "pipfile"`` is our analog + # of pip's ``direct``. See divergence note above. + is_pipfile = any( + getattr(row.requirement, "source", None) == "pipfile" + for row in info_rows + ) + + # Step 3: walk every ``SpecifierSet`` attached to any of the + # requirement rows and break it into a list of ``(operator, + # version)`` tuples — mirrors pip's ``operators`` comprehension + # at provider.py lines 230-234. + operators: list[tuple[str, str]] = [] + for row in info_rows: + spec_set = getattr(row.requirement, "specifier", None) + if spec_set is None: + continue + operators.extend( + (spec.operator, spec.version) for spec in spec_set + ) + + # Step 4: derive pinned / upper-bounded / unfree from the + # operator list — mirrors provider.py lines 236-241 verbatim. + # ``op[:2] == "=="`` covers both ``==`` and ``===`` (the latter + # is pip's arbitrary-equality, also pin-shaped). + is_pinned = any( + (op[:2] == "==") and ("*" not in ver) + for op, ver in operators + ) + is_upper_bounded = any( + (op in ("<", "<=", "~=")) or (op == "==" and "*" in ver) + for op, ver in operators + ) + is_unfree = bool(operators) + + # Step 5: backtrack-cause count for this identifier. Pip + # routes this through ``narrow_requirement_selection`` + + # ``_conflict_promoted`` (see divergence note above); we render + # the raw count here, lower = better. Match on the identifier + # tuple via ``self.identify`` so a ``RequirementInformation`` + # whose requirement has the same ``(name, extras)`` counts. + backtrack_count = 0 + for row in backtrack_causes: + req = getattr(row, "requirement", None) + if req is None: + continue + try: + row_id = self.identify(req) + except Exception: + # Defensive: a malformed requirement in the backtrack + # list shouldn't crash preference computation — skip + # it and let resolution proceed. ``resolvelib`` + # invariants say this can't happen, but pip applies + # the same defensiveness around ``ireqs`` parsing. + continue + if row_id == identifier: + backtrack_count += 1 + + # Step 6: identifier name for the lexicographic tie-break. Our + # identifier is a ``(name, frozenset(extras))`` tuple — we sort + # on ``name`` first then ``sorted(extras)`` so the order is + # deterministic across runs. Same intent as pip's trailing + # ``identifier`` slot. + name, extras = identifier + identifier_key = (name, tuple(sorted(extras))) + + # Promote-to-front: identifiers that have already caused + # backtracking (either in the current cycle via + # ``backtrack_count > 0`` or persistently via + # :attr:`_conflict_promoted`, populated by + # :meth:`narrow_requirement_selection`) get a False here + # (sorts before True), matching pip's ``not conflict_promoted`` + # slot. Backtracking is expensive when it propagates through a + # wide unconflicted layer; resolving high-conflict identifiers + # earlier prunes the search tree. + has_backtracked = ( + backtrack_count > 0 or identifier in self._conflict_promoted + ) + + return ( + not has_backtracked, + not is_pipfile, + not is_pinned, + not is_upper_bounded, + not is_unfree, + identifier_key, + ) + + # ------------------------------------------------------------------ + # T_PARITY_REAL extras-work follow-up (2026-05-13) + # — narrow_requirement_selection: focus the resolver on + # conflict-prone identifiers per pip's + # ``PipProvider.narrow_requirement_selection``. + # ------------------------------------------------------------------ + + # Mirror pip's ``_CONFLICT_PRIORITY_THRESHOLD`` + # (``pipenv/patched/pip/_internal/resolution/resolvelib/provider.py``) + # — an identifier needs to appear as an unresolved backtrack cause + # at least this many times before it gets promoted persistently. + _CONFLICT_PRIORITY_THRESHOLD = 8 + + def narrow_requirement_selection( + self, + identifiers: Iterable[Identifier], + resolutions: Mapping[Identifier, Any], + candidates: Mapping[Identifier, Iterable[Any]], + information: Mapping[Identifier, Iterable[Any]], + backtrack_causes: Sequence[Any], + ) -> Iterable[Identifier]: + """Return a subset of ``identifiers`` to focus on first. + + Mirrors pip's + :meth:`PipProvider.narrow_requirement_selection` algorithm + (``pipenv/patched/pip/_internal/resolution/resolvelib/provider.py`` + line 120) — called O(1) times per backtrack step (vs. + :meth:`get_preference` which is O(n)) so it's the right place + to do filtering that involves walking ``backtrack_causes``. + + Three-tier strategy, lifted verbatim from pip: + + 1. **Active backtrack causes** — if any + ``backtrack_causes`` row points at an identifier that's + also in the current selection AND not yet pinned, return + ONLY those identifiers. Forces the resolver to surface + the conflict immediately rather than pinning unrelated + packages first. + + 2. **Conflict-promoted** — identifiers that have crossed the + :attr:`_CONFLICT_PRIORITY_THRESHOLD` (i.e. have been a + backtrack cause that many times) stay promoted across + backtrack steps. Returned next. + + 3. **Default** — return the full ``identifiers`` iterable + unchanged. + + Bench-fixture trigger (2026-05-13): the + ``sentry-kafka-schemas`` ↔ ``sentry-protos`` ↔ ``protobuf`` + ↔ ``grpcio`` ↔ ``grpcio-status`` constraint web that the + extras-roundtrip fix surfaces. Without narrowing, + ``get_preference`` is called per-identifier and the resolver + thrashes through dozens of unrelated candidates before + learning to pin the conflict packages first. Narrowing + focuses the search. + """ + # Resolvelib hands us a fresh iterator each call — materialise + # so we can pre-flight checks AND fall through to "all" + # without re-iterating a one-shot view. + identifiers = list(identifiers) + if not identifiers: + return identifiers + + # Step 1: walk backtrack_causes, track per-identifier conflict + # counts, populate ``_conflict_promoted`` once an identifier + # crosses the threshold. Mirrors pip lines 142-152. + backtrack_idents: set[Identifier] = set() + for info in backtrack_causes: + req = getattr(info, "requirement", None) + parent = getattr(info, "parent", None) + for source in (req, parent): + if source is None: + continue + try: + name = self.identify(source) + except Exception: # noqa: BLE001 + continue + backtrack_idents.add(name) + # Only bump the persistent counter for identifiers + # that are NOT already pinned — pinning resolves a + # conflict; only unresolved ones deserve promotion. + if name not in resolutions: + self._conflict_counts[name] += 1 + if ( + self._conflict_counts[name] + >= self._CONFLICT_PRIORITY_THRESHOLD + ): + self._conflict_promoted.add(name) + + # Step 2: collect the live backtrack causes that are also + # currently-considered identifiers AND already-promoted + # identifiers. Mirrors pip lines 154-166. + current_backtrack_causes: list[Identifier] = [] + promoted: list[Identifier] = [] + for identifier in identifiers: + if identifier in backtrack_idents: + current_backtrack_causes.append(identifier) + continue + if identifier in self._conflict_promoted: + promoted.append(identifier) + + # Step 3: prefer the live backtrack causes; fall back to + # persistently-promoted; finally the full list. Pip's + # exact precedence at lines 168-174. + if current_backtrack_causes: + return current_backtrack_causes + if promoted: + return promoted + return identifiers + + # ------------------------------------------------------------------ + # T6 — is_satisfied_by (final-acceptance predicate) + # ------------------------------------------------------------------ + + def is_satisfied_by( + self, + requirement: Requirement, + candidate: Any, + ) -> bool: + """Return ``True`` iff ``candidate`` satisfies ``requirement``. + + This is the predicate ``resolvelib.AbstractProvider`` calls when + a candidate has been chosen for a requirement and the resolver + needs final confirmation it's a valid pick. Signature from + ``pipenv/patched/pip/_vendor/resolvelib/providers.py:127``:: + + def is_satisfied_by(self, requirement, candidate) -> bool: ... + + Three checks (per design §5.3 / plan T6): + + 1. **Version**: ``candidate.version in requirement.specifier`` + via :meth:`SpecifierSet.contains`. Prerelease policy mirrors + T4's :meth:`find_matches` (admit when either the specifier + opts in or :attr:`_allow_prereleases` is set). + + 2. **Extras compatibility**: every extra in + ``requirement.extras`` must be a subset of + ``candidate.extras`` (which models + ``provides_extras``) — BUT when the candidate's + ``extras`` is empty we treat it as "metadata not yet loaded" + and admit the candidate. This mirrors pip's + :meth:`SpecifierRequirement.is_satisfied_by` at + ``pipenv/patched/pip/_internal/resolution/resolvelib/requirements.py:111`` + which checks ONLY the specifier — pip relies on the + ``(name, extras)`` identifier grouping (see + :meth:`identify`) to prevent ``django`` candidates from + ever being matched against ``django[argon2]`` requirements. + In Phase 3 we keep the same lazy-metadata stance: the + METADATA file is only fetched by T7's + :meth:`get_dependencies` (the expensive path), so at the + point ``is_satisfied_by`` runs, ``provides_extras`` is + typically unknown. Once a future caller does populate the + field, this method strict-checks. See parity-divergence + note recorded on the T6 plan entry. + + 3. **Marker**: when ``requirement.marker`` is not ``None``, + evaluate it against the resolver's :attr:`_target_env` + (overriding :func:`pipenv.vendor.packaging.markers.default_environment`). + ``marker is None`` passes unconditionally. + + Pip source reference for the audit trail + ---------------------------------------- + Pip's ``PipProvider.is_satisfied_by`` + (``pipenv/patched/pip/_internal/resolution/resolvelib/provider.py:300``) + is a one-liner ``return requirement.is_satisfied_by(candidate)`` + delegating to the concrete ``Requirement`` subclass — for + ``SpecifierRequirement`` (the common case) that's just the + specifier check at ``requirements.py:111``. Pip does NOT + evaluate markers here either: marker filtering happens upstream + when ``iter_dependencies`` builds the requirement list. Our + :class:`Requirement` doesn't pre-filter by marker (T7 will when + building transitive requirements), so we evaluate the marker + defensively at the predicate. Records as a parity-divergence + candidate for T_PARITY_MATRIX — strictly more conservative than + pip's behaviour (admits a strict subset of what pip admits). + """ + # ------------ Check 1: version ------------ + try: + version_obj = Version(candidate.version) + except InvalidVersion: + # Same loud-failure stance as T4: an unparseable version in + # the cache is an upstream bug, and silently admitting it + # here would mask it. + return False + + spec = requirement.specifier + # Prerelease policy at the predicate: mirror pip's + # :meth:`SpecifierRequirement.is_satisfied_by` at + # ``pipenv/patched/pip/_internal/resolution/resolvelib/requirements.py:121`` + # — pass ``prereleases=True`` unconditionally. Pip's rationale + # (paraphrased from the comment there): ``PackageFinder`` + # filtered prereleases out upstream, so by the time the + # predicate runs, an arriving prerelease candidate was already + # admitted by the prerelease policy. Our :meth:`find_matches` + # plays the ``PackageFinder`` role here (T4 already filters + # prereleases via :attr:`_allow_prereleases` + each specifier's + # ``prereleases`` flag), so this method should not re-apply the + # policy and silently reject a candidate :meth:`find_matches` + # legitimately handed back. + if not spec.contains(version_obj, prereleases=True): + return False + + # ------------ Check 2: extras compatibility ------------ + # ``candidate.extras`` is the closest analog to METADATA's + # ``Provides-Extra``. Empty frozenset → metadata not loaded; + # admit per the lazy-metadata clause above. Non-empty → + # strict subset check. + candidate_extras = getattr(candidate, "extras", frozenset()) + if candidate_extras and not requirement.extras <= candidate_extras: + return False + + # ------------ Check 3: marker ------------ + marker = requirement.marker + if marker is not None: + env = self._marker_environment() + try: + if not marker.evaluate(env): + return False + except Exception: + # A malformed marker shouldn't crash resolution — pip + # surfaces this as a warning and treats the marker as + # "doesn't apply". We adopt the same defensive + # stance: treat unevaluable markers as not-satisfied + # so the candidate is rejected rather than crashing. + return False + + return True + + def _marker_environment(self) -> dict[str, Any]: + """Build the environment dict used to evaluate + :class:`Marker` instances on requirements. + + Starts from :func:`pipenv.vendor.packaging.markers.default_environment` + (running-Python defaults) and overlays + ``self._target_env`` so the resolver evaluates markers against + the *target* Python rather than the running Python. This is + what lets a CI host running Python 3.13 resolve a lockfile + targeting 3.10. + + Only used by :meth:`is_satisfied_by` today; T7's + :meth:`get_dependencies` will reuse it to filter transitive + requirements by marker. + """ + from pipenv.vendor.packaging.markers import default_environment + + env: dict[str, Any] = dict(default_environment()) + if isinstance(self._target_env, Mapping): + env.update(self._target_env) + return env + + # ------------------------------------------------------------------ + # T7 — get_dependencies (transitive requirement expansion) + # ------------------------------------------------------------------ + + def get_dependencies(self, candidate: Any) -> list[Requirement]: + """Return the candidate's transitive :class:`Requirement` set. + + Signature contract — :class:`AbstractProvider` from + ``pipenv/patched/pip/_vendor/resolvelib/providers.py:138``:: + + def get_dependencies(self, candidate) -> Iterable[Requirement]: ... + + Algorithm (per design §5.3 + plan T7, updated in Phase 3b T_S3): + + 1. **Metadata fetch (wheel OR sdist)**: invoke + ``self._metadata_fetcher(candidate)`` — a caller-supplied + callable that returns a + :class:`pipenv.resolver.pure_python_metadata.CoreMetadata`. + T9 wires the production stack so that this callable is a + thin closure around :func:`fetch_metadata`, which itself + routes wheel candidates to RANGE-fetched ``METADATA`` files + and sdist candidates through T_S1's PEP 517 build path — + both producing the same :class:`CoreMetadata` shape. Tests + supply a stub mapping ``candidate.url`` → ``CoreMetadata``. + + Phase 3a (pre-T_S3) had a fail-loud guard here that raised + :class:`_SdistEncountered` on non-wheel candidates and + T9 translated to :class:`InternalError`; that guard is gone + in Phase 3b — sdists now resolve transparently. The + :class:`_SdistEncountered` class remains importable for + back-compat (see its docstring). + 2. **Parse + filter** each entry in ``metadata.requires_dist``: + - Parse via :class:`pipenv.vendor.packaging.requirements.Requirement` + — that's the packaging parser (NOT this module's T1 + :class:`Requirement` dataclass; the parser yields + ``name / extras / specifier / marker`` which we then + translate INTO our T1 model). + - Skip the entry if its marker is non-``None`` and evaluates + False under the candidate's extras (see below). + - Otherwise build a new :class:`Requirement` with + ``source="transitive"`` and + ``parent=``. + 3. **Return** as a list. ``resolvelib`` accepts any iterable; + list is cheapest to consume and avoids accidental + single-pass-iterator bugs in downstream callers. + + Marker semantics (parent-extras context) + ---------------------------------------- + We mirror pip's three-branch logic at + ``pipenv/patched/pip/_internal/metadata/importlib/_dists.py:224``:: + + if not req.marker: + yield req + elif not extras and req.marker.evaluate({"extra": ""}): + yield req + elif any(req.marker.evaluate({"extra": e}) for e in extras): + yield req + + i.e. the requirement's marker is evaluated against a marker + environment that includes ``extra``: when the parent candidate + requested no extras, ``extra=""`` is the context; when it + requested ``[dev]``, ``extra="dev"`` is the context. This is + what makes ``Requires-Dist: pytest; extra=='dev'`` correctly + survive only when the parent was ``django[dev]``. + + Non-extra markers (e.g. ``python_version < '3.8'``) are + evaluated against the same overlay; pip and we both rely on + the marker's ``extra=`` clause being a top-level conjunct, so + an env with ``extra=""`` doesn't accidentally satisfy a + ``python_version`` marker that's actually False. + + Parity-divergence note for T_PARITY_MATRIX + ------------------------------------------ + Pip ALSO synthesises a "depends on the exact base" requirement + (``factory.make_requirement_from_candidate(self.base)`` at + ``candidates.py:533``) when expanding an + :class:`ExtrasCandidate`. We don't model :class:`ExtrasCandidate` + as a separate node yet (T1's :class:`Requirement` carries + ``extras`` directly and our :meth:`identify` partitions on + ``(name, extras)``); the equivalent constraint emerges from the + ``(name, frozenset())`` identifier sharing candidates with + ``(name, frozenset({"dev"}))`` via the cache. Records as a + candidate divergence for the T_PARITY_MATRIX doc; lockfile + byte-identity in T15 / T_PARITY_REAL is the gate. + + Pip ALSO swallows malformed ``Requires-Dist`` lines at + ``_dists.py:224`` (the ``get_requirement`` call raises + :class:`InvalidRequirement`; pip doesn't catch). We propagate + the same exception — a malformed METADATA body is a wheel-side + bug worth surfacing rather than silently dropping a real dep. + """ + # --------------- Metadata fetch (wheel OR sdist) ----------- + # Phase 3b T_S3: dropped the ``if not candidate.is_wheel: raise + # _SdistEncountered(candidate)`` guard. Sdist candidates now + # resolve transparently because T_S2's + # :meth:`MetadataFetcher.fetch_metadata` branches on + # ``candidate.is_wheel`` and routes sdists through T_S1's PEP + # 517 builder. ``self._metadata_fetcher`` is a callable (see + # __init__ docstring); T9 binds a session + cache around T2's + # :func:`fetch_metadata`; tests pass a dict-backed stub. + metadata = self._metadata_fetcher(candidate) + + # Canonicalise the parent name once — multiple Requires-Dist + # entries will share it. Mirrors T1's + # :meth:`Requirement.from_pipfile_entry` behaviour + # (canonicalisation at construction time so resolvelib's + # ``(name, extras)`` identifier groups by canonical name). + parent_name = canonicalize_name(getattr(candidate, "name", "")) + + # Marker environment: the SAME overlay used by T6's + # :meth:`is_satisfied_by` (built from + # :func:`packaging.markers.default_environment` overlaid with + # ``self._target_env``). The ``extra=`` slot is added per + # requirement below. + base_env = self._marker_environment() + + # Parent extras (frozenset) — drives the marker-context fork. + # Empty frozenset → use ``extra=""`` context (single iteration). + parent_extras: frozenset[str] = getattr( + candidate, "extras", frozenset() + ) + + deps: list[Requirement] = [] + + # Synthetic base-version requirement (mirrors pip's + # :class:`ExtrasCandidate.iter_dependencies` shape at + # ``pipenv/patched/pip/_internal/resolution/resolvelib/candidates.py`` + # line 533 — ``yield factory.make_requirement_from_candidate(self.base)``). + # When the parent candidate carries extras, emit a tie-down + # FIRST that pins the BARE ``(name, frozenset())`` identifier + # to the exact version of THIS candidate. Without this, the + # bare and extras-flavoured identifiers explore version space + # independently and backtracking explodes when any unrelated + # transitive pulls the bare name (sentry-bench fixture: many + # google-cloud-* packages pull ``google-api-core[grpc]`` + # while the user pulled bare ``google-api-core>=2.15.0``, + # producing two streams that diverge on every backtrack). + # Pip yields this FIRST, before the extras-gated transitives, + # so resolvelib sees the version pin before exploring the + # cascade — order matters for convergence speed. + cand_version = getattr(candidate, "version", None) + if parent_extras and cand_version: + deps.append( + Requirement( + name=parent_name, + specifier=SpecifierSet(f"=={cand_version}"), + extras=frozenset(), + marker=None, + source="transitive", + parent=parent_name, + ) + ) + + for raw_spec in metadata.requires_dist: + # ``packaging.requirements.Requirement`` parses the line. + # We pre-strip leading/trailing whitespace defensively; + # CoreMetadata already does this on construction + # (pure_python_metadata.py:592-596), but a stray empty + # line in a hand-crafted test fixture shouldn't crash the + # parser. + raw_spec = raw_spec.strip() + if not raw_spec: + continue + parsed = PackagingRequirement(raw_spec) + + # Marker filter — mirror pip's _dists.py:230-235. + if parsed.marker is not None: + if not self._marker_active_for_extras( + parsed.marker, base_env, parent_extras + ): + continue + + # Translate parser-side fields into our T1 dataclass. + # ``packaging``'s ``.extras`` is a ``set`` (mutable); freeze + # so the dataclass's ``__hash__`` is well-defined. + # + # Dual-marker note (Initiative G Phase 3b, T_M2): + # We populate BOTH ``marker`` and ``introducing_marker`` + # with the parser's ``.marker`` because the two fields play + # different roles downstream: + # * ``marker`` — the constraint-side marker used by T6's + # :meth:`is_satisfied_by` when evaluating a candidate + # against this transitive. The legacy T7 test contract + # (``test_extra_marker_kept_when_parent_requested_that_extra``) + # pins this to the parser's marker on transitives. + # * ``introducing_marker`` — the Requires-Dist-side marker + # read by T_M3's ``_translate_mapping`` to emit a + # ``markers="..."`` clause on the lockfile entry so the + # pure-python lockfile output matches pip. + # The two fields could in principle be unified, but keeping + # them distinct preserves the existing T7 contract while + # letting T_M3 evolve marker-canonicalisation independently + # of the resolver's evaluator semantics. + # Strip any ``extra == X`` / ``extra != X`` clauses from + # the marker before attaching it to the resulting + # :class:`Requirement`. The extras-gating role has already + # been consumed at this point (the requirement only made it + # past :meth:`_marker_active_for_extras` because the + # parent's extras context satisfied the clause), and + # keeping the clause on the runtime marker means T6's + # :meth:`is_satisfied_by` would re-evaluate it under the + # plain target env (no ``extra`` key) and reject every + # candidate — pip dodges this by emitting requirements + # without the extras clause once the gate has passed. + # ``introducing_marker`` keeps the ORIGINAL marker so the + # lockfile emitter (T_M3 :func:`_introducing_marker_for`) + # still records the precise ``Requires-Dist`` text. + runtime_marker = ( + _strip_extra_clauses(parsed.marker) + if parsed.marker is not None + else None + ) + deps.append( + Requirement( + name=canonicalize_name(parsed.name), + specifier=parsed.specifier, + extras=frozenset(parsed.extras), + marker=runtime_marker, + source="transitive", + parent=parent_name, + introducing_marker=parsed.marker, + ) + ) + + return deps + + def _marker_active_for_extras( + self, + marker: Any, + base_env: Mapping[str, Any], + parent_extras: frozenset[str], + ) -> bool: + """Return ``True`` iff ``marker`` evaluates True for the parent's + extras context (mirrors pip's _dists.py:230-235). + + Branch logic: + + * **No parent extras** (``parent_extras == frozenset()``): + evaluate once under ``{"extra": ""}``. This is what makes a + plain ``Requires-Dist: foo; python_version<'3.8'`` survive + (the marker has no ``extra==`` clause and ``extra=""`` is + irrelevant to it) while a ``Requires-Dist: foo; extra=='dev'`` + is filtered out (the marker's ``extra=='dev'`` is False under + ``extra=""``). + * **Parent has extras**: evaluate the marker once per extra, + ORing the results. Pip uses ``any(...)`` over + ``[{"extra": e} for e in extras]``; we match exactly. + + Defensive: a malformed marker (e.g. references an unknown + marker variable) is treated as inactive — same loud-failure + stance as T6's :meth:`is_satisfied_by` (better to surface the + issue via "this dep didn't apply" than to crash mid-resolve). + Pip silently ignores the same shape per + :meth:`Marker.evaluate`'s "missing key" path. + """ + # Build a fresh environment per evaluation so we don't mutate + # the caller's ``base_env`` (which is shared across all + # Requires-Dist lines in a single get_dependencies call). + if not parent_extras: + env = dict(base_env) + env["extra"] = "" + try: + return bool(marker.evaluate(env)) + except Exception: # noqa: BLE001 + return False + + for extra in parent_extras: + env = dict(base_env) + env["extra"] = extra + try: + if marker.evaluate(env): + return True + except Exception: # noqa: BLE001 + # Continue trying other extras — a marker that's + # malformed under one extra context may still evaluate + # successfully under another (rare but possible). + continue + return False + + +# --------------------------------------------------------------------------- +# T8 — _drive_resolver: thin wrapper around resolvelib.Resolver +# --------------------------------------------------------------------------- + + +def _drive_resolver( + requirements: Iterable[Requirement], + provider: PurePythonProvider, + *, + reporter: Any = None, + max_rounds: int = 200_000, +) -> Any: + """Convenience wrapper around :class:`resolvelib.Resolver` for + unit and integration tests. + + Production code (T9's :class:`PurePythonBackend`) calls this helper + too, so the wiring of provider → reporter → :meth:`Resolver.resolve` + lives in exactly one place. Translation of + :class:`ResolutionImpossible` into the backend's + :class:`ResolverResponse` shape is T9's concern — this helper + deliberately does not catch the exception. (Phase 3b T_S3 removed + :class:`_SdistEncountered` from the production path; sdist + METADATA is now built transparently inside the provider's + :meth:`get_dependencies` via T_S1+T_S2.) + + Parameters + ---------- + requirements: + Iterable of top-level :class:`Requirement` instances (typically + the parsed Pipfile entries). Consumed eagerly inside resolvelib; + no need to re-yield. + provider: + Configured :class:`PurePythonProvider`. All five + ``AbstractProvider`` methods (T3–T7) must be functional — + :func:`_drive_resolver` does not patch around half-implemented + providers. + reporter: + Optional :class:`resolvelib.BaseReporter` subclass (or + compatible duck-type). Defaults to a freshly-instantiated + :class:`BaseReporter` (no-op callbacks) when ``None`` is passed. + T9 may pass a logging reporter; tests use the default. + max_rounds: + Forwarded verbatim to :meth:`Resolver.resolve`. Mirrors pip's + ``limit_how_complex_resolution_can_be = 200_000`` from + :mod:`pip._internal.resolution.resolvelib.resolver` — the + 100-round resolvelib default trips on real-world Pipfiles + (the ~100-package bench fixture exhausts it inside the first + category) and pip-backend parity demands the same headroom. + Raise only if a real lock genuinely needs more rounds — + bumping further usually masks a circular-dep bug upstream. + + Returns + ------- + A :class:`resolvelib.resolvers.abstract.Result` namedtuple with + ``.mapping`` (resolved candidates keyed on the provider's + identifier tuple), ``.graph`` (dep DAG), and ``.criteria`` + (per-identifier resolution metadata). Callers typically only need + ``.mapping``. + + Raises + ------ + :class:`resolvelib.ResolutionImpossible` + Propagated from :meth:`Resolver.resolve` when no satisfying + assignment exists. T9 catches and translates. + """ + from pipenv.patched.pip._vendor.resolvelib import BaseReporter, Resolver + + if reporter is None: + reporter = BaseReporter() + return Resolver(provider, reporter).resolve( + requirements, max_rounds=max_rounds + ) diff --git a/pipenv/resolver/pure_python_requirement.py b/pipenv/resolver/pure_python_requirement.py new file mode 100644 index 0000000000..c8faa4527c --- /dev/null +++ b/pipenv/resolver/pure_python_requirement.py @@ -0,0 +1,228 @@ +"""Typed ``Requirement`` model for the pure-Python resolver backend +(Initiative G Phase 3, T1). + +Represents one constraint in the resolution graph as a frozen dataclass +— replaces pip's ``InstallRequirement`` for the in-tree +``resolvelib.Provider`` path. + +A :class:`Requirement` is what the provider hands to ``resolvelib`` as +a graph node alongside :class:`pipenv.resolver.candidate.Candidate` +(the matching artifact). ``resolvelib`` groups requirements by +``(name, extras)`` via :meth:`PurePythonProvider.identify` (T3), so the +dataclass exposes ``name`` PEP-503-canonical (lowercase, ``-`` +separators) and ``extras`` as a :class:`frozenset` — both hashable +and stable across construction sites. + +The class is intentionally minimal: it carries the *constraints* +declared by either the Pipfile or a transitive's ``Requires-Dist`` line, +nothing else. Candidate-side concerns (URL, hashes, wheel tags) live +on :class:`Candidate`; satisfaction checks (``is_satisfied_by``) and +candidate ordering (``find_matches``) live on :class:`PurePythonProvider` +(T4–T6). + +Critical constraint (enforced by Phase 1's pre-commit grep gate): +**this module must not import from patched-pip's internal package.** +The vendored ``pipenv.vendor.packaging`` is permitted and supplies +:class:`SpecifierSet` and :class:`Marker`. + +See ``docs/dev/initiative-g-phase3-design.md`` §5.1 for the design +brief and ``initiative-g-phase3-plan.md`` T1 for the validation +matrix. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Literal + +from pipenv.vendor.packaging.markers import Marker +from pipenv.vendor.packaging.specifiers import SpecifierSet +from pipenv.vendor.packaging.utils import canonicalize_name + +__all__ = ["Requirement"] + + +# Allowed ``source`` values. Kept as a ``Literal`` on the dataclass +# field so static checkers catch typos; the runtime accepts any string +# (the dataclass is frozen but doesn't enforce ``Literal`` membership — +# pyright / mypy do). +SourceLiteral = Literal["pipfile", "transitive", "constraint"] + + +@dataclass(frozen=True, slots=True) +class Requirement: + """One constraint in the resolution graph. + + Mirrors design §5.1. All fields are immutable (``frozen=True``) + and laid out compactly (``slots=True``) — phase-3 resolves carry + hundreds to low-thousands of these in memory, so the footprint + matters. + + Field semantics + --------------- + name: + PEP 503 canonical (lowercase, ``-`` separators). Use + :meth:`from_pipfile_entry` to construct from a Pipfile shape — + it runs ``canonicalize_name`` for you. Direct construction + assumes the caller already canonicalised. + specifier: + :class:`SpecifierSet` from ``pipenv.vendor.packaging``. An + empty :class:`SpecifierSet` (``SpecifierSet("")``) means "any + version is acceptable" — this is what a Pipfile ``"*"`` entry + flattens to. + extras: + :class:`frozenset` of extra names this requirement requests. + ``resolvelib``'s identifier is ``(name, extras)`` — two + requests for the same package with different extras are + distinct graph nodes. Frozenset (not list/tuple) so the + dataclass-derived ``__hash__`` is order-independent. + marker: + Optional :class:`Marker` from ``pipenv.vendor.packaging``. + ``None`` means "always applies". Evaluation against the + target environment happens in :meth:`PurePythonProvider.is_satisfied_by` + (T6) and dependency-filtering inside + :meth:`PurePythonProvider.get_dependencies` (T7). + source: + Where this requirement came from — a Pipfile entry, a + transitive ``Requires-Dist``, or a constraint file. Drives + :meth:`PurePythonProvider.get_preference`'s tie-breaking + (T5): Pipfile-direct beats transitive. + parent: + Name of the candidate that produced this transitive + requirement (``None`` for Pipfile / constraint sources). Used + for diagnostics — when a conflict arises ``resolvelib`` reports + the offending requirement and the parent chain is what the + user needs to understand "why is this in my graph?". + introducing_marker: + Optional :class:`Marker` carried over from the ``Requires-Dist`` + line that introduced this transitive — for example, + ``Requires-Dist: pytest; python_version < '3.10'`` produces a + ``Marker("python_version < '3.10'")`` on the transitive + :class:`Requirement` even after the existing marker-evaluator + filter has decided to keep the entry under the target + environment. T_M2 populates the slot in + :meth:`PurePythonProvider.get_dependencies`; T_M3 reads it in + ``_translate_mapping`` to emit a ``markers="..."`` clause on + the resulting :class:`LockedRequirement` so the pure-python + lockfile matches pip's marker output (Initiative G Phase 3b). + ``None`` (the default) means the introducing ``Requires-Dist`` + line carried no marker — or the requirement isn't a + transitive at all (Pipfile / constraint sources). + + The slot is deliberately *not* exposed via + :meth:`from_pipfile_entry` — that helper only builds top-level + Pipfile constraints (which never carry an introducing marker + by definition); transitives are constructed via the direct + ``Requirement(...)`` call inside the provider. + + Hashability + ----------- + :class:`SpecifierSet` and :class:`Marker` are both hashable in + ``pipenv.vendor.packaging`` (specifiers.py:842, markers.py:329) — + the dataclass-derived ``__hash__`` from ``frozen=True`` therefore + works out of the box for every field on the class, including the + new :attr:`introducing_marker` slot (also a :class:`Marker | None`). + Phase-3 callers rely on ``frozenset[Requirement]`` membership, so + this is verified explicitly by the T1 / T11 / T_M1 test suite. + """ + + name: str + specifier: SpecifierSet + extras: frozenset[str] + marker: Marker | None + source: SourceLiteral + parent: str | None = None + # Introduced in Initiative G Phase 3b (T_M1) — populated for + # transitives by T_M2, read by T_M3 in ``_translate_mapping``. + # Defaults to ``None`` so existing call sites keep working without + # changes; ``from_pipfile_entry`` does not expose this kwarg + # because top-level Pipfile entries never carry an introducing + # marker. + introducing_marker: Marker | None = None + + @classmethod + def from_pipfile_entry( + cls, + name: str, + value: Any, + *, + source: SourceLiteral = "pipfile", + parent: str | None = None, + ) -> Requirement: + """Build a :class:`Requirement` from a Pipfile-shape entry. + + Handles the three canonical Pipfile shapes: + + * ``"*"`` (any version) → empty :class:`SpecifierSet`. + * ``">=4.0,<6"`` (version specifier string) → parsed + :class:`SpecifierSet`. + * ``{"version": ">=4.0", "extras": ["argon2"], + "markers": "python_version >= '3.10'"}`` → all three fields + parsed. ``"version": "*"`` also flattens to an empty + :class:`SpecifierSet`. + + ``name`` is canonicalised via + :func:`pipenv.vendor.packaging.utils.canonicalize_name` — + ``"Django_Rest"`` → ``"django-rest"`` — so callers don't have + to pre-normalise. + + Other dict keys (``editable``, ``path``, ``git``, ``index``, + etc.) are *not* interpreted here: they belong on + :class:`Candidate` / a separate VCS-source path, not on the + resolution-graph constraint node. T1's job is the constraint + triple; richer source kinds land in later phases. + + Parameters + ---------- + name: + Package name as it appears in the Pipfile (any casing). + value: + Either a string version specifier or a dict-form Pipfile + entry. + source: + ``"pipfile"`` (default), ``"transitive"``, or + ``"constraint"``. T5's ``get_preference`` reads this. + parent: + Name of the parent candidate for transitives, else + ``None``. + """ + canonical = canonicalize_name(name) + + if isinstance(value, str): + spec_string = value + extras_iter: Any = () + markers_string: str | None = None + elif isinstance(value, dict): + spec_string = value.get("version", "*") + extras_iter = value.get("extras") or () + markers_string = value.get("markers") + else: + # Unknown shape — refuse loudly rather than producing a + # half-populated constraint. Pipfile parsing upstream + # should only hand us str/dict; anything else is a bug. + raise TypeError( + f"unsupported Pipfile entry value for {name!r}: " + f"{type(value).__name__}" + ) + + if spec_string in (None, "", "*"): + specifier = SpecifierSet("") + else: + specifier = SpecifierSet(spec_string) + + extras = frozenset(str(e) for e in extras_iter) + + marker: Marker | None + if markers_string: + marker = Marker(markers_string) + else: + marker = None + + return cls( + name=canonical, + specifier=specifier, + extras=extras, + marker=marker, + source=source, + parent=parent, + ) diff --git a/pipenv/resolver/pure_python_sdist.py b/pipenv/resolver/pure_python_sdist.py new file mode 100644 index 0000000000..770247ab68 --- /dev/null +++ b/pipenv/resolver/pure_python_sdist.py @@ -0,0 +1,590 @@ +"""Sdist ``METADATA`` extractor for the pure-Python resolver backend +(Initiative G Phase 3b, T_S1). + +Phase 3a deliberately failed loud (Q-A "fail loud") when a candidate +was an sdist with no wheel companion. Phase 3b inverts that: we now +build the sdist's METADATA on the fly via PEP 517's +:class:`pyproject_hooks.BuildBackendHookCaller`, so the pure-python +backend handles any package pip would. + +Flow per call to :func:`extract_metadata_from_sdist`: + +1. **Cache** lookup by ``candidate.url``. Sdist URLs are immutable + on PyPI so cache entries are valid forever; corruption is silently + treated as a miss (caller refetches and overwrites). +2. **Download** the sdist body via the duck-typed ``session`` shared + with :mod:`pipenv.resolver.pure_python_metadata`. We use the same + urllib3-style ``.request("GET", url, headers=, timeout=)`` shape so + tests can swap a :class:`unittest.mock.MagicMock` and production + passes the same configured session the wheel-METADATA path uses. +3. **Extract** the archive into a separate tempdir. ``.tar.gz``, + ``.tar.bz2``, ``.tar.xz``, plain ``.tar`` and ``.zip`` are + supported (the union of what PyPI sdists actually use). Path + traversal is blocked: member names containing ``..`` segments or + absolute paths are rejected. Sdist convention requires exactly + one top-level directory; we enforce it. +4. **Locate** ``pyproject.toml``. If present, parse the + ``[build-system]`` table for ``build-backend`` (and the optional + ``backend-path``). If absent or no backend declared, fall back to + the PEP 517 §10 legacy default ``setuptools.build_meta:__legacy__``. +5. **Drive** the PEP 517 ``prepare_metadata_for_build_wheel`` hook + inside an isolated build env created by :mod:`build` (the PyPA + PEP 517 frontend). ``DefaultIsolatedEnv`` spins up a throwaway + venv, installs the project's ``[build-system].requires`` plus any + ``get_requires_for_build_wheel`` extras, then invokes the backend. + Without this isolation the hook subprocess uses pipenv's own + interpreter, which doesn't carry package-specific build backends + like ``poetry-core`` or ``hatchling`` — surfaced when the bench + fixture hit ``python3-saml`` and ``redis-py-cluster``. + +6. **Timeout** the build at 300 seconds via + :mod:`concurrent.futures`. A wedged backend (rare but observed on + packages with C-extension probes that hang on broken CI runners) + surfaces as :class:`SdistBuildError` rather than blocking resolve. +7. **Read + parse** the ``METADATA`` file out of the dist-info dir + the backend just produced; parse via + :func:`pure_python_metadata._parse_metadata_text` so wheel and sdist + results share the exact same :class:`CoreMetadata` shape. +8. **Cache.put** if a cache is provided. +9. **Cleanup**: every tempdir is wrapped in a context manager so a + crash or timeout leaves no detritus on disk. + +Critical constraint (enforced by the T17 pre-commit gate): +**this module must not import from patched-pip's internal package.** +Importing :mod:`pipenv.patched.pip._vendor.pyproject_hooks` is +permitted (``_vendor``, not ``_internal``); that's the whole point of +re-using patched-pip's vendored frontend rather than re-vendoring it. +""" + +from __future__ import annotations + +import concurrent.futures +import hashlib +import logging +import tarfile +import tempfile +import zipfile +from pathlib import Path +from typing import Any + +try: # py3.11+ + import tomllib as _toml_loader +except ImportError: # py3.10 fallback — pipenv supports >=3.10 + from pipenv.patched.pip._vendor import tomli as _toml_loader # type: ignore[no-redef] + +from pipenv.resolver.pure_python_metadata import ( + CoreMetadata, + MetadataCache, + MetadataFetchError, + _http_request, + _parse_metadata_text, + _response_body, + _response_status, +) + +__all__ = [ + "SdistBuildError", + "extract_metadata_from_sdist", +] + +_LOGGER = logging.getLogger(__name__) + +# PEP 517 §10 legacy fallback: a sdist with no ``pyproject.toml`` (or +# one without a ``[build-system].build-backend`` entry) is built via +# the setuptools legacy shim, which knows how to drive a setup.py. +_LEGACY_BACKEND = "setuptools.build_meta:__legacy__" + +# Cap on how long we wait for the PEP 517 hook to return before +# bailing out. Five minutes matches the design doc's risk-mitigation +# entry; in practice prepare_metadata_for_build_wheel on a healthy +# sdist returns in well under a second. +_BUILD_TIMEOUT_SECONDS = 300.0 + + +class SdistBuildError(MetadataFetchError): + """Raised when sdist METADATA extraction fails. + + Subclasses :class:`MetadataFetchError` so the upstream + :class:`MetadataFetcher` (T_S2) can keep a single ``except`` clause + and surface the failure uniformly with wheel-side errors. The + underlying cause — download failure, archive corruption, build + backend traceback, or timeout — is preserved via ``__cause__`` + where applicable so logs / stack traces remain debuggable. + """ + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def extract_metadata_from_sdist( + candidate: Any, + session: Any, + *, + cache: MetadataCache | None = None, +) -> CoreMetadata: + """Build ``candidate`` via PEP 517 and return its :class:`CoreMetadata`. + + Parameters + ---------- + candidate: + Any object with ``.url`` (str) and ``.filename`` (str) + attributes. In production this is a + :class:`pipenv.resolver.candidate.Candidate`; tests can pass a + ``SimpleNamespace`` to avoid the full dataclass construction. + session: + Duck-typed HTTP session matching the + :mod:`pipenv.resolver.pure_python_metadata` shape: + ``session.request(method, url, *, headers=, timeout=)`` → + response with ``.status`` / ``.data`` (urllib3 shape) or + ``.status_code`` / ``.content`` (requests shape). + cache: + Optional read-through cache. Hits bypass HTTP + build + entirely; misses get a ``cache.put`` after a successful build. + + Returns + ------- + CoreMetadata + The parsed metadata, indistinguishable in shape from a wheel + result, so downstream resolver logic doesn't need to branch. + + Raises + ------ + SdistBuildError + On any unrecoverable failure: HTTP non-2xx / no-body, archive + corruption, path traversal, missing/invalid build backend, + backend traceback, or build timeout. + """ + if cache is not None: + cached = cache.get(candidate.url) + if cached is not None: + return cached + + # The two tempdirs (download + extracted source) are independent; + # both must be cleaned up regardless of which step fails. We use + # nested context managers so a crash in extraction still frees the + # download dir. + with tempfile.TemporaryDirectory(prefix="pipenv-sdist-dl-") as dl_dir: + archive_path = _download_sdist(candidate, session, Path(dl_dir)) + + with tempfile.TemporaryDirectory(prefix="pipenv-sdist-src-") as src_dir: + source_root = _extract_sdist(archive_path, Path(src_dir)) + + backend, backend_path = _resolve_build_backend(source_root) + + with tempfile.TemporaryDirectory( + prefix="pipenv-sdist-meta-" + ) as meta_dir: + metadata = _run_prepare_metadata( + source_root, + backend, + backend_path, + Path(meta_dir), + ) + + if cache is not None: + try: + cache.put(candidate.url, metadata) + except OSError as exc: + # Cache write failure is non-fatal — we have the metadata + # in hand. Mirror the pure_python_metadata.fetch_metadata + # contract so callers don't have to special-case sdists. + _LOGGER.debug( + "sdist metadata cache write failed for %s: %s", + candidate.url, + exc, + ) + + return metadata + + +# --------------------------------------------------------------------------- +# Step 1: download +# --------------------------------------------------------------------------- + + +def _download_sdist(candidate: Any, session: Any, dest_dir: Path) -> Path: + """GET ``candidate.url`` and write the body to a file in ``dest_dir``. + + Returns the on-disk archive path. Raises :class:`SdistBuildError` + on any non-2xx response, empty body, or hash mismatch — all are + non-recoverable for the extraction step that follows. + + If the candidate carries SHA-256 hashes (PEP 691 / PyPI simple API), + the downloaded body is verified against them before writing to disk. + A mismatch raises :class:`SdistBuildError` to prevent executing + tampered artifacts in the PEP 517 build hooks. + """ + url = candidate.url + response = _http_request(session, "GET", url) + if response is None: + raise SdistBuildError( + f"sdist download failed: GET {url} returned no response" + ) + status = _response_status(response) + if status not in (200, 206): + raise SdistBuildError( + f"sdist download failed: GET {url} returned HTTP {status}" + ) + body = _response_body(response) + if body is None: + raise SdistBuildError( + f"sdist download failed: GET {url} returned empty body" + ) + + # Hash verification: if the candidate advertises SHA-256 hashes + # (PEP 691 / PyPI simple API), verify the downloaded body before + # writing it to disk. A mismatch is a security boundary — we + # MUST NOT execute a tampered sdist's PEP 517 hooks. + expected_hashes = getattr(candidate, "hashes", None) or {} + sha256_hashes = expected_hashes.get("sha256", []) + if sha256_hashes: + computed = hashlib.sha256(bytes(body)).hexdigest() + if computed not in sha256_hashes: + raise SdistBuildError( + f"sdist download failed: SHA-256 mismatch for {url} " + f"(computed={computed}, expected one of {sha256_hashes})" + ) + + # Filename comes from candidate.filename — same source the wheel + # path uses — so the file's extension survives intact and the + # extractor in step 2 can dispatch on it. + archive_name = getattr(candidate, "filename", None) or _filename_from_url(url) + # Sanitise: strip any directory components so a malformed simple-API + # filename containing path separators or an absolute path cannot + # write outside the temporary directory. + archive_name = Path(archive_name).name + # Path("..").name == ".." and Path(".").name == ".", so these checks + # are reachable: an index returning "filename=.." would pass the + # Path.name strip but still write to a confusing location. + if not archive_name.strip() or archive_name in (".", ".."): + raise SdistBuildError( + f"sdist download failed: invalid archive filename derived from {url!r}" + ) + archive_path = dest_dir / archive_name + try: + archive_path.write_bytes(bytes(body)) + except OSError as exc: + raise SdistBuildError( + f"sdist download failed: could not write {archive_path}: {exc}" + ) from exc + return archive_path + + +def _filename_from_url(url: str) -> str: + """Last path segment of ``url``, sans query string.""" + tail = url.rsplit("/", 1)[-1] + return tail.split("?", 1)[0] or "sdist.tar.gz" + + +# --------------------------------------------------------------------------- +# Step 2: extract +# --------------------------------------------------------------------------- + + +def _extract_sdist(archive_path: Path, dest_dir: Path) -> Path: + """Extract ``archive_path`` into ``dest_dir`` and return the source root. + + Sdist convention (PEP 517 §6 + PEP 643): the archive contains + exactly one top-level directory named ``{name}-{version}`` whose + contents are the source tree. We enforce that invariant and + return the path to that directory. + + Member-name validation rejects any path that: + + * is absolute, or + * contains a ``..`` segment, or + * resolves outside ``dest_dir`` after join+normalize. + + This blocks the classic ``tar`` traversal attack + (``../../../etc/passwd``) and the less-obvious symlink-after-extract + variant — we just refuse the archive outright. + """ + name = archive_path.name.lower() + try: + if name.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tbz2", + ".tar.xz", ".txz", ".tar")): + _extract_tar(archive_path, dest_dir) + elif name.endswith(".zip"): + _extract_zip(archive_path, dest_dir) + else: + # Unknown extension — try tar; tarfile.open auto-detects + # the compression for the common cases. If that fails + # we surface a clear error. + try: + _extract_tar(archive_path, dest_dir) + except (tarfile.TarError, OSError) as exc: + raise SdistBuildError( + f"sdist archive corrupt: unsupported extension on {name}" + ) from exc + except SdistBuildError: + raise + except (tarfile.TarError, zipfile.BadZipFile, EOFError, OSError) as exc: + raise SdistBuildError( + f"sdist archive corrupt: {exc}" + ) from exc + + return _locate_source_root(dest_dir, archive_path.name) + + +def _extract_tar(archive_path: Path, dest_dir: Path) -> None: + with tarfile.open(archive_path, "r:*") as tf: + members = tf.getmembers() + for m in members: + _validate_member_name(m.name, archive_path.name) + # Reject device files, fifos, symlinks pointing outside, + # etc. Only regular files + directories are part of a + # well-formed sdist. + if m.islnk() or m.issym(): + _validate_member_name(m.linkname, archive_path.name) + if m.isdev() or m.ischr() or m.isfifo() or m.isblk(): + raise SdistBuildError( + f"sdist archive corrupt: {archive_path.name} contains " + f"a non-regular member {m.name!r}" + ) + # Python 3.12+ deprecated the implicit ``filter`` arg; pass + # ``data`` explicitly so 3.12 / 3.13 don't warn and 3.14 + # doesn't break. ``data`` filter is the safe-by-default one. + try: + tf.extractall(dest_dir, filter="data") + except TypeError: + # Older Python (3.10 / early 3.11) without the ``filter`` + # kwarg — our manual validation above already covered the + # traversal cases. + tf.extractall(dest_dir) + + +def _extract_zip(archive_path: Path, dest_dir: Path) -> None: + with zipfile.ZipFile(archive_path, "r") as zf: + for info in zf.infolist(): + _validate_member_name(info.filename, archive_path.name) + zf.extractall(dest_dir) + + +def _validate_member_name(member_name: str, archive_name: str) -> None: + """Reject member names that would escape the extraction root. + + ``..`` segments and absolute paths are both fatal. We don't try + to be clever and re-anchor them — the archive is malformed and + we refuse it outright. + """ + if not member_name: + # Empty member names are nonsensical but happen in the wild + # (zips produced by some legacy tooling). Reject explicitly + # rather than letting ``Path("")`` quietly produce ``.``. + raise SdistBuildError( + f"sdist archive corrupt: {archive_name} contains an empty member name" + ) + # Normalise separators so a Windows-style ``..\\foo`` is caught on Linux. + norm = member_name.replace("\\", "/") + if norm.startswith("/"): + raise SdistBuildError( + f"sdist archive corrupt: {archive_name} contains an absolute " + f"member path {member_name!r}" + ) + for part in norm.split("/"): + if part == "..": + raise SdistBuildError( + f"sdist archive corrupt: {archive_name} contains a path " + f"traversal segment in {member_name!r}" + ) + + +def _locate_source_root(dest_dir: Path, archive_name: str) -> Path: + """Find the single top-level directory created by extraction. + + Sdist convention: exactly one ``{name}-{version}/`` directory at + the archive root. Anything else (zero entries, multiple entries, + or a top-level file rather than directory) is a malformed sdist + and we surface :class:`SdistBuildError`. + """ + entries = [e for e in dest_dir.iterdir() if not e.name.startswith(".")] + # Filter out PaxHeaders pseudo-entries that some tar implementations + # produce at the archive root. Those are not part of the source + # tree. + entries = [e for e in entries if "PaxHeader" not in e.name] + if not entries: + raise SdistBuildError( + f"sdist archive corrupt: {archive_name} extracted to nothing" + ) + dirs = [e for e in entries if e.is_dir()] + if len(dirs) != 1: + raise SdistBuildError( + f"sdist archive corrupt: {archive_name} must contain exactly " + f"one top-level directory, found {len(dirs)} " + f"({[d.name for d in dirs]})" + ) + return dirs[0] + + +# --------------------------------------------------------------------------- +# Step 3: resolve the build backend +# --------------------------------------------------------------------------- + + +def _resolve_build_backend(source_root: Path) -> tuple[str, list[str] | None]: + """Return ``(build_backend, backend_path)`` for ``source_root``. + + Reads ``pyproject.toml`` if present; otherwise applies the PEP 517 + §10 legacy fallback. A ``pyproject.toml`` that's present but + malformed surfaces as :class:`SdistBuildError` — a malformed + pyproject means the sdist is broken and we can't build it. + """ + pyproject = source_root / "pyproject.toml" + if not pyproject.is_file(): + return _LEGACY_BACKEND, None + try: + raw = pyproject.read_bytes() + config = _toml_loader.loads(raw.decode("utf-8")) + except (OSError, UnicodeDecodeError, ValueError) as exc: + raise SdistBuildError( + f"build backend failed: could not read pyproject.toml: {exc}" + ) from exc + # tomllib raises TOMLDecodeError (subclass of ValueError) — caught above. + + build_system = config.get("build-system") if isinstance(config, dict) else None + if not isinstance(build_system, dict): + return _LEGACY_BACKEND, None + + backend = build_system.get("build-backend") + if not isinstance(backend, str) or not backend: + # Per PEP 517 §10: pyproject.toml without a build-backend + # uses the legacy default. + return _LEGACY_BACKEND, None + + raw_path = build_system.get("backend-path") + backend_path: list[str] | None + if isinstance(raw_path, list) and all(isinstance(p, str) for p in raw_path): + backend_path = list(raw_path) + else: + backend_path = None + + return backend, backend_path + + +# --------------------------------------------------------------------------- +# Step 4: drive the PEP 517 hook +# --------------------------------------------------------------------------- + + +def _build_metadata_in_isolated_env( + source_root: Path, metadata_dir: Path +) -> Path: + """Build the project's ``.dist-info`` in a throwaway PEP 517 venv. + + Returns the on-disk path to the produced ``.dist-info`` directory + (the directory itself, not the ``METADATA`` file inside it). + + Extracted as a module-level function so tests can monkeypatch it + in lieu of orchestrating a real isolated venv (which would add + multi-second setup overhead per test). Production callers go + through :func:`_run_prepare_metadata` which wraps this in a + timeout-bounded :class:`ThreadPoolExecutor`. + + **Known limitation**: the isolated environment installs + ``[build-system].requires`` using pip's default index configuration + (PyPI). For sdists whose build backend dependencies live on a + private index or require custom sources, metadata extraction will + fail or fall back to PyPI. Threading the resolver request's source + configuration into this build-isolation install path is deferred to + a future phase to avoid coupling the sdist extractor to the + resolver's request schema. + + :mod:`build` is imported lazily so wheel-only resolves never pay + the import cost. + """ + from pipenv.vendor.build import ProjectBuilder + from pipenv.vendor.build.env import DefaultIsolatedEnv + + with DefaultIsolatedEnv() as env: + builder = ProjectBuilder.from_isolated_env(env, str(source_root)) + env.install(builder.build_system_requires) + # ``get_requires_for_build`` accepts ``"sdist"`` or ``"wheel"``; + # metadata extraction shares the wheel-build dependency set in + # PEP 517, so install those too before calling ``metadata_path``. + env.install(builder.get_requires_for_build("wheel")) + return Path(builder.metadata_path(str(metadata_dir))) + + +def _run_prepare_metadata( + source_root: Path, + backend: str, + backend_path: list[str] | None, + metadata_dir: Path, +) -> CoreMetadata: + """Drive PEP 517 ``prepare_metadata_for_build_wheel`` in an isolated env. + + Uses :class:`build.env.DefaultIsolatedEnv` to create a throwaway + venv, installs the project's ``[build-system].requires`` plus any + ``get_requires_for_build_wheel`` extras into it, then invokes the + backend's ``prepare_metadata_for_build_wheel`` hook. Without the + isolation step, package-specific build backends (poetry-core, + hatchling, flit-core, …) crash on import because pipenv's own + interpreter doesn't carry them — that surfaced when the bench + fixture hit ``python3-saml`` and ``redis-py-cluster``. + + The ``backend`` and ``backend_path`` arguments are accepted for + error-reporting symmetry with :func:`_resolve_build_backend` but + the canonical pyproject-table read is performed inside + :class:`build.ProjectBuilder` itself, so any divergence between the + two reads would point at a ``build``-version mismatch worth + surfacing. + + Wrapped in a :class:`ThreadPoolExecutor` with a hard 300-second + timeout — :mod:`build` itself has no timeout knob. The subprocess + the hook caller spawns continues running on timeout (we can't kill + it without a runner swap), but using ``shutdown(wait=False)`` in + a ``finally`` clause means we return to the caller immediately + rather than blocking for the full duration of a wedged build. The + background thread is a daemon thread that will be reaped when the + process exits. + """ + pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) + # Note: ThreadPoolExecutor uses daemon threads by default, so any + # worker that outlives this function (e.g. on timeout) will be + # reaped automatically when the process exits. + try: + future = pool.submit( + _build_metadata_in_isolated_env, source_root, metadata_dir + ) + try: + dist_info_path = future.result(timeout=_BUILD_TIMEOUT_SECONDS) + except concurrent.futures.TimeoutError as exc: + raise SdistBuildError( + f"sdist build timed out after {int(_BUILD_TIMEOUT_SECONDS)}s " + f"(backend={backend!r})" + ) from exc + except Exception as exc: + # Backend hook failures (BackendUnavailable, HookMissing, + # CalledProcessError) and env-setup failures (BuildException + # / BuildBackendException from :mod:`build`) all flatten + # into a single typed error so callers don't have to learn + # the :mod:`build` exception hierarchy. The original + # exception stays attached via ``__cause__``. + raise SdistBuildError( + f"build backend failed: {backend!r}: {exc}" + ) from exc + finally: + # Always shut down without waiting: if the future completed + # normally there is nothing pending to wait for; if it timed out + # or raised, blocking here would defeat the timeout entirely. + pool.shutdown(wait=False) + + metadata_path = dist_info_path / "METADATA" + try: + raw = metadata_path.read_bytes() + except OSError as exc: + raise SdistBuildError( + f"build backend failed: METADATA file not produced at " + f"{metadata_path}: {exc}" + ) from exc + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise SdistBuildError( + f"build backend failed: METADATA at {metadata_path} is not " + f"UTF-8: {exc}" + ) from exc + + return _parse_metadata_text(text) diff --git a/pipenv/routines/install.py b/pipenv/routines/install.py index 160afe6309..f0e6bd3404 100644 --- a/pipenv/routines/install.py +++ b/pipenv/routines/install.py @@ -1,5 +1,7 @@ +import logging import os import queue +import shutil import sys import warnings from collections import defaultdict @@ -32,6 +34,8 @@ from pipenv.utils.project import ensure_project from pipenv.utils.shell import temp_environ +_LOGGER = logging.getLogger(__name__) + def _target_marker_environment(project, allow_global=False): """Build a marker environment dict reflecting the venv's Python version. @@ -840,6 +844,7 @@ def batch_install( """ target = ctx.target_env exec_opts = ctx.execution_options + policy = ctx.install_policy if sequential_deps is None: sequential_deps = [] @@ -898,51 +903,96 @@ def batch_install( trusted_hosts=get_trusted_hosts(), pypi_mirror=target.pypi_mirror, ) + + # Parallel wheel pre-fetch — fan out the wheel downloads across a + # 16-connection urllib3 pool BEFORE handing the requirements list + # to pip. Pip downloads serially, so on a cold cache the sentry- + # base bench's 151 wheels cost ~12 s of network in the install + # subprocess; doing them in parallel here cuts that to ~3 s and + # pip's ``--find-links `` reuse path is a local stat / copy. + # Best-effort: a ``None`` return (no successful pre-fetches — + # nothing to share, network down, lockfile lacks hashes, etc.) + # leaves the regular index-driven pip-install path untouched. + # + # Only honour the pre-fetch when we're running from a hash-pinned + # lockfile (``policy.skip_lock`` is False) — without hashes there + # is no authoritative match key to verify the downloaded bytes + # against, and we MUST NOT mutate pip's install with unverified + # wheel files. + find_links_dir: str | None = None + if not policy.skip_lock and isinstance(lockfile_section, dict): + try: + from pipenv.utils.prefetch import prefetch_wheels + + find_links_dir = prefetch_wheels( + project, + deps_to_install, + lockfile_section, + sources, + allow_global=target.allow_global, + ) + except Exception as exc: # noqa: BLE001 — defensive; never block install + _LOGGER.debug("prefetch_wheels raised, skipping: %s", exc) + find_links_dir = None + + iter_extra_pip_args = list(extra_pip_args) + if find_links_dir: + iter_extra_pip_args += ["--find-links", find_links_dir] + # Build a per-call ctx that carries the (possibly-augmented) # ``extra_pip_args`` down to ``batch_install_iteration``. iter_ctx = replace( ctx, - execution_options=replace(exec_opts, extra_pip_args=tuple(extra_pip_args)), + execution_options=replace( + exec_opts, extra_pip_args=tuple(iter_extra_pip_args) + ), ) - if search_all_sources: - dependencies = [pip_line for _, pip_line in deps_to_install] - batch_install_iteration( - project, - iter_ctx, - dependencies, - sources, - procs, - requirements_dir, - ) - else: - # Sort the dependencies out by index -- include editable/vcs in the default group - deps_by_index = defaultdict(list) - for dependency, pip_line in deps_to_install: - index = project.sources.default["name"] - if dependency.name and dependency.name in lockfile_section: - entry = lockfile_section[dependency.name] - if isinstance(entry, dict) and "index" in entry: - index = entry["index"] - deps_by_index[index].append(pip_line) - # Treat each index as its own pip install phase - for index_name, dependencies in deps_by_index.items(): - try: - install_source = next(filter(lambda s: s["name"] == index_name, sources)) - batch_install_iteration( - project, - iter_ctx, - dependencies, - [install_source], - procs, - requirements_dir, - ) - except StopIteration: # noqa: PERF203 - console.print( - f"Unable to find {index_name} in sources, please check dependencies: {dependencies}", - style="bold red", - ) - sys.exit(1) + try: + if search_all_sources: + dependencies = [pip_line for _, pip_line in deps_to_install] + batch_install_iteration( + project, + iter_ctx, + dependencies, + sources, + procs, + requirements_dir, + ) + else: + # Sort the dependencies out by index -- include editable/vcs in the default group + deps_by_index = defaultdict(list) + for dependency, pip_line in deps_to_install: + index = project.sources.default["name"] + if dependency.name and dependency.name in lockfile_section: + entry = lockfile_section[dependency.name] + if isinstance(entry, dict) and "index" in entry: + index = entry["index"] + deps_by_index[index].append(pip_line) + # Treat each index as its own pip install phase + for index_name, dependencies in deps_by_index.items(): + try: + install_source = next(filter(lambda s: s["name"] == index_name, sources)) + batch_install_iteration( + project, + iter_ctx, + dependencies, + [install_source], + procs, + requirements_dir, + ) + except StopIteration: # noqa: PERF203 + console.print( + f"Unable to find {index_name} in sources, please check dependencies: {dependencies}", + style="bold red", + ) + sys.exit(1) + finally: + # Clean up the prefetch temp directory after pip has consumed it. + # The directory is created by prefetch_wheels() with tempfile.mkdtemp() + # and must be removed to avoid leaving pipenv-prefetch-* dirs in /tmp. + if find_links_dir: + shutil.rmtree(find_links_dir, ignore_errors=True) def _cleanup_procs(project, procs): diff --git a/pipenv/routines/lock.py b/pipenv/routines/lock.py index e3bd3b92c8..6c31dde05c 100644 --- a/pipenv/routines/lock.py +++ b/pipenv/routines/lock.py @@ -280,12 +280,41 @@ def do_lock(project, ctx: RoutineContext): pre = policy.pre quiet = exec_opts.quiet write = exec_opts.write - resolver = exec_opts.resolver extra_pip_args = ( list(exec_opts.extra_pip_args) if exec_opts.extra_pip_args else None ) categories = list(sel.categories) if sel.categories else None + # T_PLUMBING (Initiative G phase 3): backend selection. + # + # Precedence (matching :func:`pipenv.resolver.core._selected_backend_name` + # but evaluated parent-side so the choice can ride on the typed + # ``ResolverRequest.options.backend`` field): + # + # 1. CLI flag (``--backend NAME`` or back-compat ``--resolver NAME``) + # → ``ctx.execution_options.resolver`` + # 2. ``[pipenv] resolver_backend`` Pipfile setting (T_PLUMBING) + # 3. ``[pipenv] resolver`` Pipfile setting (T_F.5 back-compat) + # 4. ``None`` (== empty wire sentinel) — the dispatcher's + # env/Pipfile/default chain in the child / in-process branch + # still applies. + # + # We do NOT default to ``"pip"`` here: the empty wire sentinel keeps + # the default-path resolve byte-identical to today (no + # ``ResolverOptions.backend`` field set on the request, so + # ``ResolverRequest.to_json_dict`` strips it). + resolver_backend = exec_opts.resolver + if not resolver_backend: + try: + resolver_backend = project.settings.get("resolver_backend") or None + except Exception: # noqa: BLE001 — Pipfile-read failure defers to default + resolver_backend = None + if not resolver_backend: + try: + resolver_backend = project.settings.get("resolver") or None + except Exception: # noqa: BLE001 + resolver_backend = None + # T17: ``--clear`` must invalidate our parsed-manifest cache in # addition to pip's HTTP/resolution caches. Runs at the top of # the lock so a cache wipe is visible to every resolve in this @@ -417,7 +446,13 @@ def do_lock(project, ctx: RoutineContext): old_lock_data=old_lock_data, extra_pip_args=extra_pip_args, resolved_default_deps=category_default_deps, - resolver_backend=resolver, + # T_PLUMBING (Initiative G phase 3): pass the resolved + # backend name down to the resolver-subprocess argv / + # in-process driver. ``None`` is the "not specified" + # sentinel; the wire-shape stays empty-string so the + # default ``--backend pip``-or-unset path is + # byte-identical to today. + resolver_backend=resolver_backend, ) except RuntimeError: sys.exit(1) diff --git a/pipenv/utils/internet.py b/pipenv/utils/internet.py index f8c59399e6..92c27af60e 100644 --- a/pipenv/utils/internet.py +++ b/pipenv/utils/internet.py @@ -247,7 +247,13 @@ def write_credentials_netrc(sources, directory) -> Optional[str]: existing = _read_existing_netrc_content().strip() body = "\n".join(machine_blocks) if existing: - body = f"{body}\n{existing}\n" + # ``netrc.authenticators()`` returns the LAST matching entry for a + # host, so our Pipfile-derived blocks must come AFTER the user's + # existing netrc; otherwise a stale system entry for the same host + # silently overrides the credentials the user just supplied via + # Pipfile env-vars (regression seen after GHSA-8xgg-v3jj-95m2 moved + # auth off argv onto netrc — gh-6670). + body = f"{existing}\n\n{body}" netrc_path = os.path.join(str(directory), "pipenv-netrc") # Write with restrictive permissions: netrc parsers (and pip's vendored diff --git a/pipenv/utils/pip.py b/pipenv/utils/pip.py index a02e5587b9..db54e0af09 100644 --- a/pipenv/utils/pip.py +++ b/pipenv/utils/pip.py @@ -146,11 +146,23 @@ def pip_install_deps( get_runnable_pip(), "install", ] + # ``--upgrade`` was hardcoded True historically; this is the + # safety-net that ensured a sync re-installation downgraded an + # already-installed package to the lockfile pin. With + # ``--no-deps`` + explicit ``pkg==X.Y.Z`` lines and the + # ``is_satisfied`` filter that already runs upstream (see + # ``batch_install``), ``--upgrade`` is now redundant on cold + # installs — pip otherwise does a per-package metadata check + # that costs measurable wall time on a clean venv with N=151 + # packages. We DO still pass ``--upgrade`` when not running + # under ``--no-deps`` so the historical behaviour of ``pipenv + # install`` (Pipfile-driven, may need to downgrade) is + # preserved. pip_args = get_pip_args( project, pre=project.settings.get("allow_prereleases", False), verbose=False, # When True, the subprocess fails to recognize the EOF when reading stdout. - upgrade=True, + upgrade=not no_deps, no_use_pep517=not use_pep517, no_deps=no_deps, extra_pip_args=extra_pip_args, diff --git a/pipenv/utils/pipfile.py b/pipenv/utils/pipfile.py index bb5bfb8d75..98a17cb876 100644 --- a/pipenv/utils/pipfile.py +++ b/pipenv/utils/pipfile.py @@ -126,6 +126,29 @@ } +_VALID_PACKAGE_NAME_CASE_MODES = ("off", "canonical", "pypi") + + +def _normalize_package_name_case(value) -> str: + """Coerce a ``[pipenv] package_name_case`` setting to one of + ``off`` / ``canonical`` / ``pypi``. ``None``, missing, ``False``, and + the string ``"false"`` all map to ``off``; any unrecognised value also + falls back to ``off`` (silent — there is no project layer here to + surface a warning, and the safe default is "do nothing"). + """ + if value is None or value is False: + return "off" + if isinstance(value, bool): + # ``True`` is permissive — treat as "do something offline". + return "canonical" + text = str(value).strip().lower() + if text in _VALID_PACKAGE_NAME_CASE_MODES: + return text + if text in ("", "false", "none", "no", "0"): + return "off" + return "off" + + def walk_up(bottom): """mimic os.walk, but walk 'up' instead of down the directory tree.""" # Convert to Path object and resolve to absolute path @@ -1177,10 +1200,60 @@ def add_packages_batch(self, packages_data, dev=False, categories=None): # ---- casing ----------------------------------------------------------- def recase(self) -> None: - """Walk packages/dev-packages and fix any incorrect casing.""" + """Walk packages/dev-packages and fix package name casing. + + Mode is read from the ``[pipenv] package_name_case`` Pipfile + setting. Three values are recognised: + + ``"off"`` (default, also when the key is absent or any falsy + string): leave names as the user wrote them. + + ``"canonical"``: apply + :func:`packaging.utils.canonicalize_name` to every entry — PEP 503 + normalization, fully offline. Fast and deterministic. + + ``"pypi"``: legacy behaviour — issue one synchronous + ``https://pypi.org/pypi//json`` probe per unknown name to + learn the project's display capitalization (see + :func:`pipenv.utils.internet.proper_case`). Adds ~33 ms × N of + sequential network latency on a fresh ``install -r``; pre-2026 + releases ran this unconditionally. + + The setting lives in the Pipfile rather than an environment + variable because it is project-policy: every contributor on the + project gets the same name-casing behaviour, instead of differing + local defaults causing each ``pipenv install`` to bounce the + Pipfile back and forth between commits. + """ + mode = _normalize_package_name_case( + self._project.settings.get("package_name_case") + ) + if mode == "off": + return + if mode == "canonical": + if self._apply_canonical_casing(): + self.write_toml(self.parsed) + return + # mode == "pypi" if self.ensure_proper_casing(): self.write_toml(self.parsed) + def _apply_canonical_casing(self) -> bool: + """PEP 503 canonical normalization across ``packages`` and ``dev-packages``.""" + from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name + + pfile = self.parsed + changed = False + for section_name in ("packages", "dev-packages"): + section = pfile.get(section_name, {}) + for original in list(section.keys()): + canonical = canonicalize_name(original) + if canonical != original: + section[canonical] = section[original] + del section[original] + changed = True + return changed + def ensure_proper_casing(self) -> bool: """Apply proper-casing across ``packages`` and ``dev-packages``.""" pfile = self.parsed diff --git a/pipenv/utils/prefetch.py b/pipenv/utils/prefetch.py new file mode 100644 index 0000000000..e0e9b76e06 --- /dev/null +++ b/pipenv/utils/prefetch.py @@ -0,0 +1,429 @@ +"""Parallel wheel pre-fetch for ``pipenv install`` / ``pipenv sync``. + +Pip's ``install`` step downloads wheels sequentially from the index — +on a clean cache that dominates cold-install wall time (12 s of pure +network for the sentry-base bench's 151 wheels, vs ~10 s for the +sequential install phase itself). This module pre-populates a local +``--find-links`` directory by downloading wheels concurrently +(``urllib3`` connection-pool + ``ThreadPoolExecutor``, capped at 16 +workers to match the pool ceiling that the pure-Python resolver +already standardised on) BEFORE pipenv invokes ``pip install``. Pip +then sees the wheels in the find-links dir and skips its own +network round-trips for the matching files. + +Design constraints +------------------ + +* **Best-effort, never raises**: any per-package failure (network + hiccup, missing wheel for the target platform, hash mismatch) + falls through silently and the regular pip-install path handles + that package via the index. pipenv MUST NOT degrade because the + pre-fetch shortcut had a problem. + +* **Hash-pinned**: every download body is SHA-256-verified against + the lockfile entry's ``hashes`` list before writing to the + find-links dir. A hash mismatch is treated as a fetch failure — + the wheel is dropped and pip downloads from the index instead. + +* **Platform-aware**: the lockfile carries hashes for ALL wheel + variants of a release (cp311-linux + cp310-macosx + ...). We + download only the wheel whose tags match the *target* Python's + ``packaging.tags.sys_tags()`` — invoking `` -c "..."`` + once per pre-fetch to ask the target interpreter directly. This + avoids the host/target tag drift trap that bites every "guess the + wheel from the running interpreter" implementation. + +* **Cache-side-effect free for pip**: pre-fetched wheels go into a + caller-managed temp dir, not pip's HTTP cache. Pip is told about + them via ``--find-links `` (kept alongside ``--index-url`` so + any missing files still resolve from the index). No mutation of + the user's pip cache. + +* **Existing resolver infrastructure reused**: the simple-API + metadata fetch goes through + :class:`pipenv.resolver.pep691.PEP691Client` + + :class:`pipenv.resolver.fetcher.ParallelFetcher`, so auth / + netrc / cert handling stays in one place (the same module already + vetted for credential leaks under GHSA-8xgg-v3jj-95m2). + +Entry point: :func:`prefetch_wheels`. Called from +:func:`pipenv.routines.install.batch_install` once the dependency +list and source list are finalised; returns the ``--find-links`` dir +or ``None`` if nothing was fetched. +""" +from __future__ import annotations + +import concurrent.futures +import hashlib +import logging +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Any, Sequence + +_LOGGER = logging.getLogger(__name__) +_MAX_WORKERS = 16 +_CONNECT_TIMEOUT = 10.0 +_READ_TIMEOUT = 60.0 + + +def prefetch_wheels( + project: Any, + deps: Sequence[Any], + lockfile_section: dict, + sources: Sequence[dict], + *, + allow_global: bool = False, + max_workers: int = _MAX_WORKERS, +) -> str | None: + """Pre-fetch wheels matching the lockfile entries to a temp dir. + + Returns the temp dir path (use as ``--find-links``) on at least one + success, or ``None`` when nothing was fetched. Every error path is + swallowed; the regular pip-install flow takes over for any package + not in the returned dir. + + Parameters + ---------- + project: + Pipenv :class:`Project` — used for the venv's Python interpreter + (target wheel tags) and the cache layout. + deps: + Iterable of ``(InstallRequirement, pip_line)`` tuples (the + same shape :func:`pipenv.routines.install.batch_install` walks). + lockfile_section: + The ``[default]`` / ``[develop]`` section of the lockfile — + looked up by canonical name to get ``version`` + ``hashes``. + sources: + The post-mirror-substitution source list (dicts with ``name``, + ``url``, ``verify_ssl``). We try each index in order; first + successful match wins. + allow_global: + Forwarded to :func:`project_python` so ``--system`` installs + target the global interpreter's tags. + max_workers: + Hard cap at 16 — matches urllib3's pool ceiling per the + existing :class:`ParallelFetcher` contract. + """ + # Local imports keep the cold-import cost of pipenv unaffected + # for callers that never hit install/sync. + try: + from pipenv.patched.pip._vendor import urllib3 + from pipenv.resolver.fetcher import ParallelFetcher + from pipenv.resolver.manifest_cache import ParsedManifestCache + from pipenv.resolver.pep691 import PEP691Client + from pipenv.utils.shell import project_python + from pipenv.vendor.packaging.utils import canonicalize_name + except Exception as exc: # noqa: BLE001 + _LOGGER.debug("prefetch_wheels: import failed (%s); skipping", exc) + return None + + max_workers = min(max_workers, _MAX_WORKERS) + if max_workers < 1: + return None + + # Step 1: build the (name, version, expected_hashes) target list + # from the lockfile. Skip anything we can't pin by SHA-256 — the + # hash is the authoritative match key downstream. + targets: list[tuple[str, str, set[str]]] = [] + for dep, _pip_line in deps: + raw_name = getattr(dep, "name", None) + if not raw_name: + continue + canonical = canonicalize_name(raw_name) + entry = lockfile_section.get(raw_name) or lockfile_section.get(canonical) + if not isinstance(entry, dict): + continue + version = (entry.get("version") or "").lstrip("=").strip() + raw_hashes = entry.get("hashes") or [] + sha256s = { + h for h in raw_hashes + if isinstance(h, str) and h.startswith("sha256:") + } + if not version or not sha256s: + continue + targets.append((canonical, version, sha256s)) + + if not targets: + return None + + index_urls = tuple(s.get("url", "") for s in sources if s.get("url")) + if not index_urls: + return None + + # Step 2: get the TARGET Python's wheel tags. Host and target may + # disagree (host 3.13 + venv 3.11 is common); the lockfile carries + # hashes for every platform's wheel so we must filter to "the wheel + # the target Python would actually install." + try: + target_python = project_python(project, system=allow_global) + except Exception as exc: # noqa: BLE001 + _LOGGER.debug("prefetch_wheels: project_python lookup failed (%s)", exc) + return None + target_tags = _query_target_tags(target_python) + if target_tags is None: + # Tag query failed — bail rather than guess and waste downloads. + return None + + # Step 3: use a requests-compatible session for PEP 691 metadata + # fetches, because PEP691Client passes per-request verify/cert + # kwargs that bare urllib3.PoolManager.request() does not accept. + # Keep urllib3 for wheel downloads so the existing download path + # remains unchanged. + cache_dir_holder = tempfile.mkdtemp(prefix="pipenv-prefetch-cache-") + download_dir = Path(tempfile.mkdtemp(prefix="pipenv-prefetch-")) + try: + import requests + + manifest_session = requests.Session() + manifest_adapter = requests.adapters.HTTPAdapter( + pool_connections=max_workers, + pool_maxsize=max_workers, + ) + manifest_session.mount("http://", manifest_adapter) + manifest_session.mount("https://", manifest_adapter) + + download_session = urllib3.PoolManager( + num_pools=max_workers, + maxsize=max_workers, + ) + try: + cache = ParsedManifestCache(Path(cache_dir_holder)) + client = PEP691Client(manifest_session) + fetcher = ParallelFetcher(client, cache, max_workers=max_workers) + + fetch_targets = [ + (idx, name) for idx in index_urls for name, _, _ in targets + ] + fetcher.populate(fetch_targets) + + # Step 4: pick the right wheel per target. Filter by target + # tags FIRST (skip incompatible wheels), then match hash + # against the lockfile. + download_plan: list[tuple[str, str, str]] = [] + for name, version, expected_hashes in targets: + picked = _pick_wheel( + name, version, expected_hashes, cache, index_urls, target_tags + ) + if picked is not None: + download_plan.append(picked) + + if not download_plan: + shutil.rmtree(download_dir, ignore_errors=True) + return None + + # Step 5: parallel download + hash verify. + successes = 0 + with concurrent.futures.ThreadPoolExecutor( + max_workers=max_workers + ) as ex: + futs = [ + ex.submit(_download_and_verify, download_session, item, download_dir) + for item in download_plan + ] + for fut in concurrent.futures.as_completed(futs): + ok = _safe_future_result(fut) + if ok: + successes += 1 + + if successes == 0: + shutil.rmtree(download_dir, ignore_errors=True) + return None + + _LOGGER.info( + "Pre-fetched %d/%d wheels in parallel to %s", + successes, + len(download_plan), + download_dir, + ) + return str(download_dir) + finally: + manifest_session.close() + download_session.clear() + finally: + # The manifest cache temp dir is throwaway scratch; the + # download_dir survives (consumed by pip via --find-links) or + # has already been cleaned in the no-success branch. + shutil.rmtree(cache_dir_holder, ignore_errors=True) + + +def _query_target_tags(target_python: str) -> frozenset[tuple[str, str, str]] | None: + """Ask the target Python for its compatible wheel tags. + + Returns a frozenset of ``(interpreter, abi, platform)`` tuples on + success, ``None`` if the subprocess invocation fails for any + reason (timeout, non-zero exit, parse error). Tuple form + sidesteps importing :class:`packaging.tags.Tag` on the host (the + target Python may not even have packaging installed). + """ + snippet = ( + "import sys\n" + "try:\n" + " from packaging.tags import sys_tags\n" + "except ImportError:\n" + " try:\n" + " from pip._vendor.packaging.tags import sys_tags\n" + " except ImportError:\n" + " sys.exit(2)\n" + "for t in sys_tags():\n" + " print(f'{t.interpreter}\\t{t.abi}\\t{t.platform}')\n" + ) + try: + proc = subprocess.run( + [target_python, "-c", snippet], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + _LOGGER.debug("target-tag subprocess failed: %s", exc) + return None + if proc.returncode != 0: + _LOGGER.debug( + "target-tag subprocess rc=%d stderr=%s", + proc.returncode, + proc.stderr[:200], + ) + return None + tags: set[tuple[str, str, str]] = set() + for line in proc.stdout.splitlines(): + parts = line.strip().split("\t") + if len(parts) == 3: + tags.add(tuple(parts)) # type: ignore[arg-type] + return frozenset(tags) if tags else None + + +def _pick_wheel( + name: str, + version: str, + expected_hashes: set[str], + cache: Any, + index_urls: Sequence[str], + target_tags: frozenset[tuple[str, str, str]], +) -> tuple[str, str, str] | None: + """Choose the wheel to download for ``(name, version)``. + + Returns ``(url, filename, expected_hash)`` on a match, ``None`` + otherwise. Walks every configured index's cached manifest and + picks the first wheel that: + + * has ``cand.version == version``, + * is_wheel and not yanked, + * has at least one platform-tag in :func:`packaging.tags.sys_tags` + of the target Python (so pip will accept it as compatible), + * declares a SHA-256 hash present in the lockfile's hash set. + """ + for index_url in index_urls: + manifest = cache.get(index_url, name) + if manifest is None: + continue + for cand in getattr(manifest, "candidates", ()) or (): + if getattr(cand, "version", None) != version: + continue + if not getattr(cand, "is_wheel", False): + continue + if getattr(cand, "yanked", False): + continue + cand_tags = getattr(cand, "wheel_tags", None) or frozenset() + cand_tag_triples = { + (str(t.interpreter), str(t.abi), str(t.platform)) + for t in cand_tags + } + if not (cand_tag_triples & target_tags): + continue + for h in getattr(cand, "hashes", ()) or (): + algo = getattr(h, "algo", None) + value = getattr(h, "value", None) + if algo != "sha256" or not value: + continue + hash_str = f"{algo}:{value}" + if hash_str in expected_hashes: + return ( + getattr(cand, "url", ""), + getattr(cand, "filename", ""), + hash_str, + ) + return None + + +def _safe_future_result(fut: concurrent.futures.Future) -> bool: + """Read a worker's result without letting the loop crash on raises. + + Worker callables already swallow their own exceptions and return + ``bool``; this is a final safety net for ``fut.result()`` itself + raising (cancellation, executor shutdown). + """ + try: + return bool(fut.result()) + except Exception as exc: # noqa: BLE001 — defensive + _LOGGER.debug("prefetch worker raised: %s", exc) + return False + + +def _download_and_verify( + session: Any, + item: tuple[str, str, str], + target_dir: Path, +) -> bool: + """Download ``item`` and write it to ``target_dir`` on hash match. + + ``item`` is ``(url, filename, expected_hash)``. Returns ``True`` on + successful download + hash match + file write, ``False`` on any + failure path. All exceptions are caught and logged at DEBUG. + """ + url, filename, expected_hash = item + if not url or not filename: + return False + try: + try: + from pipenv.patched.pip._vendor.urllib3 import Timeout as Urllib3Timeout + _timeout = Urllib3Timeout(connect=_CONNECT_TIMEOUT, read=_READ_TIMEOUT) + except ImportError: + _timeout = (_CONNECT_TIMEOUT, _READ_TIMEOUT) + response = session.request( + "GET", + url, + timeout=_timeout, + preload_content=True, + ) + except Exception as exc: # noqa: BLE001 + _LOGGER.debug("prefetch GET %s failed: %s", url, exc) + return False + try: + status = getattr(response, "status", None) or getattr( + response, "status_code", 0 + ) + if int(status) != 200: + _LOGGER.debug("prefetch GET %s returned HTTP %s", url, status) + return False + data = getattr(response, "data", None) + if data is None: + data = getattr(response, "content", None) + if data is None: + return False + body = bytes(data) + actual_hash = f"sha256:{hashlib.sha256(body).hexdigest()}" + if actual_hash != expected_hash: + _LOGGER.debug( + "prefetch hash mismatch for %s: got %s, expected %s", + filename, + actual_hash, + expected_hash, + ) + return False + # Sanitise filename — defensive against an index returning a + # filename containing path separators. + safe_name = Path(filename).name + if not safe_name or safe_name in (".", ".."): + return False + (target_dir / safe_name).write_bytes(body) + return True + finally: + release = getattr(response, "release_conn", None) + if callable(release): + try: + release() + except Exception: # noqa: BLE001 + pass diff --git a/pipenv/utils/pylock.py b/pipenv/utils/pylock.py index 4390b4d929..31e8b07883 100644 --- a/pipenv/utils/pylock.py +++ b/pipenv/utils/pylock.py @@ -671,7 +671,20 @@ def convert_to_pipenv_lockfile(self) -> Dict[str, Any]: # Add sources if present if "sources" in self.data: - lockfile["_meta"]["sources"] = self.data["sources"] + # Expand ${VAR} tokens in source URLs at read time so that + # callers downstream (resolver, finder, netrc-writer) see the + # real credentials. Mirrors the legacy ``Pipfile.lock`` reader + # (``Lockfile.load``); without this, users with ``[pipenv] + # use_pylock = true`` lose env-var expansion in private-index + # URLs (gh-6670). + from pipenv.utils.shell import expand_url_credentials + + sources = [] + for source in self.data["sources"]: + if isinstance(source, dict) and "url" in source: + source = {**source, "url": expand_url_credentials(source["url"])} + sources.append(source) + lockfile["_meta"]["sources"] = sources # If no sources in pylock.toml, add a default source else: lockfile["_meta"]["sources"] = [ diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 7984baddb0..d86ded1b48 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -1140,9 +1140,24 @@ def clean_results(self) -> List[Dict[str, Any]]: def _generate_resolution_cache_key( - deps, project, pipfile_category, pre, clear, allow_global, pypi_mirror, extra_pip_args + deps, + project, + pipfile_category, + pre, + clear, + allow_global, + pypi_mirror, + extra_pip_args, + resolver_backend=None, ): - """Generate a cache key for resolution results.""" + """Generate a cache key for resolution results. + + T_PLUMBING (Initiative G phase 3): ``resolver_backend`` participates + in the key so a pip-resolved cache entry can't satisfy a + pure-python lookup. ``None`` collapses to the empty string so the + default-backend cache key is byte-identical to pre-T_PLUMBING (the + pre-existing on-disk caches are still valid for the default path). + """ # Get lockfile and pipfile modification times lockfile_mtime = "no-lock" if project.lockfile.location: @@ -1179,6 +1194,18 @@ def _generate_resolution_cache_key( str(allow_global), "|".join(env_factors), ] + # T_PLUMBING (Initiative G phase 3): backend selection + # participates in the cache key — but ONLY when a non-default + # backend is selected. Appending unconditionally would change + # the cache key for every default-path call (the empty string + # would still hash differently than the no-component shape), + # which would invalidate pre-T_PLUMBING on-disk caches and break + # the "default path byte-identical" acceptance criterion. + # Also include the PIPENV_RESOLVER env var so that a pure-python + # lookup via env var never reuses a pip-backend cache entry. + effective_backend = (resolver_backend or os.environ.get("PIPENV_RESOLVER") or "").strip() + if effective_backend: + key_components.append(f"backend={effective_backend}") key_string = "|".join(key_components) return hashlib.sha256(key_string.encode()).hexdigest() @@ -1945,7 +1972,17 @@ def venv_resolve_deps( extra_pip_args = list(extra_pip_args or []) + selected_backend = _selected_backend_for_request(project, resolver_backend) + # Preserve pre-T_PLUMBING cache-key shape for the default path: + # default "pip" remains unstamped (None) for byte-identical keys. + cache_backend = selected_backend if selected_backend != "pip" else None + # Check cache before expensive resolution + # T_PLUMBING (Initiative G phase 3): include the effective + # backend (CLI/env/Pipfile precedence) in the cache key so a + # pip-resolved cache entry can't satisfy a pure-python lookup + # (and vice versa). Keep the default "pip" path un-stamped so + # the cache key is byte-identical to pre-T_PLUMBING. cache_key = _generate_resolution_cache_key( deps, project, @@ -1955,6 +1992,7 @@ def venv_resolve_deps( allow_global, pypi_mirror, extra_pip_args, + resolver_backend=cache_backend, ) if _should_use_resolution_cache(cache_key, clear): @@ -2028,7 +2066,11 @@ def venv_resolve_deps( extra_pip_args=extra_pip_args, resolved_default_deps=resolved_default_deps, project=project, - resolver_backend=resolver_backend, + # T_PLUMBING (Initiative G phase 3): stamp the + # parent-resolved backend name onto the typed request + # so both the subprocess and in-process branches see + # the same selection. + resolver_backend=selected_backend, ) # Useful for debugging and hitting breakpoints in the resolver diff --git a/pipenv/utils/settings.py b/pipenv/utils/settings.py index 4c6d475c80..ee1deee588 100644 --- a/pipenv/utils/settings.py +++ b/pipenv/utils/settings.py @@ -202,3 +202,25 @@ def resolver(self) -> str | None: return None text = str(value).strip() return text or None + + @property + def resolver_backend(self) -> str | None: + """Return the configured resolver backend name from + ``[pipenv] resolver_backend = "..."`` in the Pipfile, or + ``None`` if unset. + + Introduced by T_PLUMBING (Initiative G phase 3, 2026-05-12). + Coexists with the T_F.5 ``[pipenv] resolver`` accessor as the + back-compat alias; the lock routine prefers + ``resolver_backend`` over ``resolver`` when both are present. + + Accepted values: ``"pip"`` (default), ``"pure-python"``. + Unknown values still produce a structured ``InternalError`` + response from the dispatcher rather than crashing — see + :func:`pipenv.resolver.core.resolve_for_pipenv`. + """ + value = self.get("resolver_backend") + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/pipenv/vendor/build/LICENSE b/pipenv/vendor/build/LICENSE new file mode 100644 index 0000000000..c3713cdcc9 --- /dev/null +++ b/pipenv/vendor/build/LICENSE @@ -0,0 +1,20 @@ +Copyright © 2019 Filipe Laíns + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next +paragraph) shall be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/pipenv/vendor/build/__init__.py b/pipenv/vendor/build/__init__.py new file mode 100644 index 0000000000..d9cba651c9 --- /dev/null +++ b/pipenv/vendor/build/__init__.py @@ -0,0 +1,39 @@ +""" +build - A simple, correct Python build frontend +""" + +from __future__ import annotations + +from ._builder import ProjectBuilder +from ._exceptions import ( + BuildBackendException, + BuildException, + BuildSystemTableValidationError, + FailedProcessError, + TypoWarning, +) +from ._types import ConfigSettings as ConfigSettingsType +from ._types import Distribution as DistributionType +from ._types import SubprocessRunner as RunnerType +from ._util import check_dependency + + +__version__ = '1.2.2' + +__all__ = [ + '__version__', + 'BuildBackendException', + 'BuildException', + 'BuildSystemTableValidationError', + 'check_dependency', + 'ConfigSettingsType', + 'DistributionType', + 'FailedProcessError', + 'ProjectBuilder', + 'RunnerType', + 'TypoWarning', +] + + +def __dir__() -> list[str]: + return __all__ diff --git a/pipenv/vendor/build/__main__.py b/pipenv/vendor/build/__main__.py new file mode 100644 index 0000000000..0217062ba1 --- /dev/null +++ b/pipenv/vendor/build/__main__.py @@ -0,0 +1,455 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import argparse +import contextlib +import contextvars +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import textwrap +import traceback +import warnings + +from collections.abc import Iterator, Sequence +from functools import partial +from typing import NoReturn, TextIO + +import pipenv.vendor.build as build + +from . import ProjectBuilder, _ctx +from . import env as _env +from ._exceptions import BuildBackendException, BuildException, FailedProcessError +from ._types import ConfigSettings, Distribution, StrPath +from .env import DefaultIsolatedEnv + + +_COLORS = { + 'red': '\33[91m', + 'green': '\33[92m', + 'yellow': '\33[93m', + 'bold': '\33[1m', + 'dim': '\33[2m', + 'underline': '\33[4m', + 'reset': '\33[0m', +} +_NO_COLORS = {color: '' for color in _COLORS} + + +_styles = contextvars.ContextVar('_styles', default=_COLORS) + + +def _init_colors() -> None: + if 'NO_COLOR' in os.environ: + if 'FORCE_COLOR' in os.environ: + warnings.warn('Both NO_COLOR and FORCE_COLOR environment variables are set, disabling color', stacklevel=2) + _styles.set(_NO_COLORS) + elif 'FORCE_COLOR' in os.environ or sys.stdout.isatty(): + return + _styles.set(_NO_COLORS) + + +def _cprint(fmt: str = '', msg: str = '', file: TextIO | None = None) -> None: + print(fmt.format(msg, **_styles.get()), file=file, flush=True) + + +def _showwarning( + message: Warning | str, + category: type[Warning], + filename: str, + lineno: int, + file: TextIO | None = None, + line: str | None = None, +) -> None: # pragma: no cover + _cprint('{yellow}WARNING{reset} {}', str(message)) + + +_max_terminal_width = shutil.get_terminal_size().columns - 2 +if _max_terminal_width <= 0: + _max_terminal_width = 78 + + +_fill = partial(textwrap.fill, subsequent_indent=' ', width=_max_terminal_width) + + +def _log(message: str, *, origin: tuple[str, ...] | None = None) -> None: + if origin is None: + (first, *rest) = message.splitlines() + _cprint('{bold}{}{reset}', _fill(first, initial_indent='* ')) + for line in rest: + print(_fill(line, initial_indent=' ')) + + elif origin[0] == 'subprocess': + initial_indent = '> ' if origin[1] == 'cmd' else '< ' + file = sys.stderr if origin[1] == 'stderr' else None + for line in message.splitlines(): + _cprint('{dim}{}{reset}', _fill(line, initial_indent=initial_indent), file=file) + + +def _setup_cli(*, verbosity: int) -> None: + warnings.showwarning = _showwarning + + if platform.system() == 'Windows': + try: + import colorama + + colorama.init() + except ModuleNotFoundError: + pass + + _init_colors() + + _ctx.LOGGER.set(_log) + _ctx.VERBOSITY.set(verbosity) + + +def _error(msg: str, code: int = 1) -> NoReturn: # pragma: no cover + """ + Print an error message and exit. Will color the output when writing to a TTY. + + :param msg: Error message + :param code: Error code + """ + _cprint('{red}ERROR{reset} {}', msg) + raise SystemExit(code) + + +def _format_dep_chain(dep_chain: Sequence[str]) -> str: + return ' -> '.join(dep.partition(';')[0].strip() for dep in dep_chain) + + +def _build_in_isolated_env( + srcdir: StrPath, + outdir: StrPath, + distribution: Distribution, + config_settings: ConfigSettings | None, + installer: _env.Installer, +) -> str: + with DefaultIsolatedEnv(installer=installer) as env: + builder = ProjectBuilder.from_isolated_env(env, srcdir) + # first install the build dependencies + env.install(builder.build_system_requires) + # then get the extra required dependencies from the backend (which was installed in the call above :P) + env.install(builder.get_requires_for_build(distribution, config_settings or {})) + return builder.build(distribution, outdir, config_settings or {}) + + +def _build_in_current_env( + srcdir: StrPath, + outdir: StrPath, + distribution: Distribution, + config_settings: ConfigSettings | None, + skip_dependency_check: bool = False, +) -> str: + builder = ProjectBuilder(srcdir) + + if not skip_dependency_check: + missing = builder.check_dependencies(distribution, config_settings or {}) + if missing: + dependencies = ''.join('\n\t' + dep for deps in missing for dep in (deps[0], _format_dep_chain(deps[1:])) if dep) + _cprint() + _error(f'Missing dependencies:{dependencies}') + + return builder.build(distribution, outdir, config_settings or {}) + + +def _build( + isolation: bool, + srcdir: StrPath, + outdir: StrPath, + distribution: Distribution, + config_settings: ConfigSettings | None, + skip_dependency_check: bool, + installer: _env.Installer, +) -> str: + if isolation: + return _build_in_isolated_env(srcdir, outdir, distribution, config_settings, installer) + else: + return _build_in_current_env(srcdir, outdir, distribution, config_settings, skip_dependency_check) + + +@contextlib.contextmanager +def _handle_build_error() -> Iterator[None]: + try: + yield + except (BuildException, FailedProcessError) as e: + _error(str(e)) + except BuildBackendException as e: + if isinstance(e.exception, subprocess.CalledProcessError): + _cprint() + _error(str(e)) + + if e.exc_info: + tb_lines = traceback.format_exception( + e.exc_info[0], + e.exc_info[1], + e.exc_info[2], + limit=-1, + ) + tb = ''.join(tb_lines) + else: + tb = traceback.format_exc(-1) + _cprint('\n{dim}{}{reset}\n', tb.strip('\n')) + _error(str(e)) + except Exception as e: # pragma: no cover + tb = traceback.format_exc().strip('\n') + _cprint('\n{dim}{}{reset}\n', tb) + _error(str(e)) + + +def _natural_language_list(elements: Sequence[str]) -> str: + if len(elements) == 0: + msg = 'no elements' + raise IndexError(msg) + elif len(elements) == 1: + return elements[0] + else: + return '{} and {}'.format( + ', '.join(elements[:-1]), + elements[-1], + ) + + +def build_package( + srcdir: StrPath, + outdir: StrPath, + distributions: Sequence[Distribution], + config_settings: ConfigSettings | None = None, + isolation: bool = True, + skip_dependency_check: bool = False, + installer: _env.Installer = 'pip', +) -> Sequence[str]: + """ + Run the build process. + + :param srcdir: Source directory + :param outdir: Output directory + :param distribution: Distribution to build (sdist or wheel) + :param config_settings: Configuration settings to be passed to the backend + :param isolation: Isolate the build in a separate environment + :param skip_dependency_check: Do not perform the dependency check + """ + built: list[str] = [] + for distribution in distributions: + out = _build(isolation, srcdir, outdir, distribution, config_settings, skip_dependency_check, installer) + built.append(os.path.basename(out)) + return built + + +def build_package_via_sdist( + srcdir: StrPath, + outdir: StrPath, + distributions: Sequence[Distribution], + config_settings: ConfigSettings | None = None, + isolation: bool = True, + skip_dependency_check: bool = False, + installer: _env.Installer = 'pip', +) -> Sequence[str]: + """ + Build a sdist and then the specified distributions from it. + + :param srcdir: Source directory + :param outdir: Output directory + :param distribution: Distribution to build (only wheel) + :param config_settings: Configuration settings to be passed to the backend + :param isolation: Isolate the build in a separate environment + :param skip_dependency_check: Do not perform the dependency check + """ + from ._compat import tarfile + + if 'sdist' in distributions: + msg = 'Only binary distributions are allowed but sdist was specified' + raise ValueError(msg) + + sdist = _build(isolation, srcdir, outdir, 'sdist', config_settings, skip_dependency_check, installer) + + sdist_name = os.path.basename(sdist) + sdist_out = tempfile.mkdtemp(prefix='build-via-sdist-') + built: list[str] = [] + if distributions: + # extract sdist + with tarfile.TarFile.open(sdist) as t: + t.extractall(sdist_out) + try: + _ctx.log(f'Building {_natural_language_list(distributions)} from sdist') + srcdir = os.path.join(sdist_out, sdist_name[: -len('.tar.gz')]) + for distribution in distributions: + out = _build(isolation, srcdir, outdir, distribution, config_settings, skip_dependency_check, installer) + built.append(os.path.basename(out)) + finally: + shutil.rmtree(sdist_out, ignore_errors=True) + return [sdist_name, *built] + + +def main_parser() -> argparse.ArgumentParser: + """ + Construct the main parser. + """ + parser = argparse.ArgumentParser( + description=textwrap.indent( + textwrap.dedent( + """ + A simple, correct Python build frontend. + + By default, a source distribution (sdist) is built from {srcdir} + and a binary distribution (wheel) is built from the sdist. + This is recommended as it will ensure the sdist can be used + to build wheels. + + Pass -s/--sdist and/or -w/--wheel to build a specific distribution. + If you do this, the default behavior will be disabled, and all + artifacts will be built from {srcdir} (even if you combine + -w/--wheel with -s/--sdist, the wheel will be built from {srcdir}). + """ + ).strip(), + ' ', + ), + # Prevent argparse from taking up the entire width of the terminal window + # which impedes readability. + formatter_class=partial(argparse.RawDescriptionHelpFormatter, width=min(_max_terminal_width, 127)), + ) + parser.add_argument( + 'srcdir', + type=str, + nargs='?', + default=os.getcwd(), + help='source directory (defaults to current directory)', + ) + parser.add_argument( + '--version', + '-V', + action='version', + version=f"build {build.__version__} ({','.join(build.__path__)})", + ) + parser.add_argument( + '--verbose', + '-v', + dest='verbosity', + action='count', + default=0, + help='increase verbosity', + ) + parser.add_argument( + '--sdist', + '-s', + dest='distributions', + action='append_const', + const='sdist', + help='build a source distribution (disables the default behavior)', + ) + parser.add_argument( + '--wheel', + '-w', + dest='distributions', + action='append_const', + const='wheel', + help='build a wheel (disables the default behavior)', + ) + parser.add_argument( + '--outdir', + '-o', + type=str, + help=f'output directory (defaults to {{srcdir}}{os.sep}dist)', + metavar='PATH', + ) + parser.add_argument( + '--skip-dependency-check', + '-x', + action='store_true', + help='do not check that build dependencies are installed', + ) + env_group = parser.add_mutually_exclusive_group() + env_group.add_argument( + '--no-isolation', + '-n', + action='store_true', + help='disable building the project in an isolated virtual environment. ' + 'Build dependencies must be installed separately when this option is used', + ) + env_group.add_argument( + '--installer', + choices=_env.INSTALLERS, + help='Python package installer to use (defaults to pip)', + ) + parser.add_argument( + '--config-setting', + '-C', + dest='config_settings', + action='append', + help='settings to pass to the backend. Multiple settings can be provided. ' + 'Settings beginning with a hyphen will erroneously be interpreted as options to build if separated ' + 'by a space character; use ``--config-setting=--my-setting -C--my-other-setting``', + metavar='KEY[=VALUE]', + ) + return parser + + +def main(cli_args: Sequence[str], prog: str | None = None) -> None: + """ + Parse the CLI arguments and invoke the build process. + + :param cli_args: CLI arguments + :param prog: Program name to show in help text + """ + parser = main_parser() + if prog: + parser.prog = prog + args = parser.parse_args(cli_args) + + _setup_cli(verbosity=args.verbosity) + + config_settings = {} + + if args.config_settings: + for arg in args.config_settings: + setting, _, value = arg.partition('=') + if setting not in config_settings: + config_settings[setting] = value + else: + if not isinstance(config_settings[setting], list): + config_settings[setting] = [config_settings[setting]] + + config_settings[setting].append(value) + + # outdir is relative to srcdir only if omitted. + outdir = os.path.join(args.srcdir, 'dist') if args.outdir is None else args.outdir + + distributions: list[Distribution] = args.distributions + if distributions: + build_call = build_package + else: + build_call = build_package_via_sdist + distributions = ['wheel'] + + with _handle_build_error(): + built = build_call( + args.srcdir, + outdir, + distributions, + config_settings, + not args.no_isolation, + args.skip_dependency_check, + args.installer, + ) + artifact_list = _natural_language_list( + ['{underline}{}{reset}{bold}{green}'.format(artifact, **_styles.get()) for artifact in built] + ) + _cprint('{bold}{green}Successfully built {}{reset}', artifact_list) + + +def entrypoint() -> None: + main(sys.argv[1:]) + + +if __name__ == '__main__': # pragma: no cover + main(sys.argv[1:], 'python -m build') + + +__all__ = [ + 'main', + 'main_parser', +] diff --git a/pipenv/vendor/build/_builder.py b/pipenv/vendor/build/_builder.py new file mode 100644 index 0000000000..5762427aab --- /dev/null +++ b/pipenv/vendor/build/_builder.py @@ -0,0 +1,355 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import contextlib +import difflib +import os +import subprocess +import sys +import warnings +import zipfile + +from collections.abc import Iterator +from typing import Any, Mapping, Sequence, TypeVar + +import pipenv.vendor.pyproject_hooks as pyproject_hooks + +from . import _ctx, env +from ._compat import tomllib +from ._exceptions import ( + BuildBackendException, + BuildException, + BuildSystemTableValidationError, + TypoWarning, +) +from ._types import ConfigSettings, Distribution, StrPath, SubprocessRunner +from ._util import check_dependency, parse_wheel_filename + + +_TProjectBuilder = TypeVar('_TProjectBuilder', bound='ProjectBuilder') + + +_DEFAULT_BACKEND = { + 'build-backend': 'setuptools.build_meta:__legacy__', + 'requires': ['setuptools >= 40.8.0'], +} + + +def _find_typo(dictionary: Mapping[str, str], expected: str) -> None: + for obj in dictionary: + if difflib.SequenceMatcher(None, expected, obj).ratio() >= 0.8: + warnings.warn( + f"Found '{obj}' in pyproject.toml, did you mean '{expected}'?", + TypoWarning, + stacklevel=2, + ) + + +def _validate_source_directory(source_dir: StrPath) -> None: + if not os.path.isdir(source_dir): + msg = f'Source {source_dir} is not a directory' + raise BuildException(msg) + pyproject_toml = os.path.join(source_dir, 'pyproject.toml') + setup_py = os.path.join(source_dir, 'setup.py') + if not os.path.exists(pyproject_toml) and not os.path.exists(setup_py): + msg = f'Source {source_dir} does not appear to be a Python project: no pyproject.toml or setup.py' + raise BuildException(msg) + + +def _read_pyproject_toml(path: StrPath) -> Mapping[str, Any]: + try: + with open(path, 'rb') as f: + return tomllib.loads(f.read().decode()) + except FileNotFoundError: + return {} + except PermissionError as e: + msg = f"{e.strerror}: '{e.filename}' " + raise BuildException(msg) from None + except tomllib.TOMLDecodeError as e: + msg = f'Failed to parse {path}: {e} ' + raise BuildException(msg) from None + + +def _parse_build_system_table(pyproject_toml: Mapping[str, Any]) -> Mapping[str, Any]: + # If pyproject.toml is missing (per PEP 517) or [build-system] is missing + # (per PEP 518), use default values + if 'build-system' not in pyproject_toml: + _find_typo(pyproject_toml, 'build-system') + return _DEFAULT_BACKEND + + build_system_table = dict(pyproject_toml['build-system']) + + # If [build-system] is present, it must have a ``requires`` field (per PEP 518) + if 'requires' not in build_system_table: + _find_typo(build_system_table, 'requires') + msg = '`requires` is a required property' + raise BuildSystemTableValidationError(msg) + elif not isinstance(build_system_table['requires'], list) or not all( + isinstance(i, str) for i in build_system_table['requires'] + ): + msg = '`requires` must be an array of strings' + raise BuildSystemTableValidationError(msg) + + if 'build-backend' not in build_system_table: + _find_typo(build_system_table, 'build-backend') + # If ``build-backend`` is missing, inject the legacy setuptools backend + # but leave ``requires`` intact to emulate pip + build_system_table['build-backend'] = _DEFAULT_BACKEND['build-backend'] + elif not isinstance(build_system_table['build-backend'], str): + msg = '`build-backend` must be a string' + raise BuildSystemTableValidationError(msg) + + if 'backend-path' in build_system_table and ( + not isinstance(build_system_table['backend-path'], list) + or not all(isinstance(i, str) for i in build_system_table['backend-path']) + ): + msg = '`backend-path` must be an array of strings' + raise BuildSystemTableValidationError(msg) + + unknown_props = build_system_table.keys() - {'requires', 'build-backend', 'backend-path'} + if unknown_props: + msg = f'Unknown properties: {", ".join(unknown_props)}' + raise BuildSystemTableValidationError(msg) + + return build_system_table + + +def _wrap_subprocess_runner(runner: SubprocessRunner, env: env.IsolatedEnv) -> SubprocessRunner: + def _invoke_wrapped_runner( + cmd: Sequence[str], cwd: str | None = None, extra_environ: Mapping[str, str] | None = None + ) -> None: + runner(cmd, cwd, {**(env.make_extra_environ() or {}), **(extra_environ or {})}) + + return _invoke_wrapped_runner + + +class ProjectBuilder: + """ + The PEP 517 consumer API. + """ + + def __init__( + self, + source_dir: StrPath, + python_executable: str = sys.executable, + runner: SubprocessRunner = pyproject_hooks.default_subprocess_runner, + ) -> None: + """ + :param source_dir: The source directory + :param python_executable: The python executable where the backend lives + :param runner: Runner for backend subprocesses + + The ``runner``, if provided, must accept the following arguments: + + - ``cmd``: a list of strings representing the command and arguments to + execute, as would be passed to e.g. 'subprocess.check_call'. + - ``cwd``: a string representing the working directory that must be + used for the subprocess. Corresponds to the provided source_dir. + - ``extra_environ``: a dict mapping environment variable names to values + which must be set for the subprocess execution. + + The default runner simply calls the backend hooks in a subprocess, writing backend output + to stdout/stderr. + """ + self._source_dir: str = os.path.abspath(source_dir) + _validate_source_directory(source_dir) + + self._python_executable = python_executable + self._runner = runner + + pyproject_toml_path = os.path.join(source_dir, 'pyproject.toml') + self._build_system = _parse_build_system_table(_read_pyproject_toml(pyproject_toml_path)) + + self._backend = self._build_system['build-backend'] + + self._hook = pyproject_hooks.BuildBackendHookCaller( + self._source_dir, + self._backend, + backend_path=self._build_system.get('backend-path'), + python_executable=self._python_executable, + runner=self._runner, + ) + + @classmethod + def from_isolated_env( + cls: type[_TProjectBuilder], + env: env.IsolatedEnv, + source_dir: StrPath, + runner: SubprocessRunner = pyproject_hooks.default_subprocess_runner, + ) -> _TProjectBuilder: + return cls( + source_dir=source_dir, + python_executable=env.python_executable, + runner=_wrap_subprocess_runner(runner, env), + ) + + @property + def source_dir(self) -> str: + """Project source directory.""" + return self._source_dir + + @property + def python_executable(self) -> str: + """ + The Python executable used to invoke the backend. + """ + return self._python_executable + + @property + def build_system_requires(self) -> set[str]: + """ + The dependencies defined in the ``pyproject.toml``'s + ``build-system.requires`` field or the default build dependencies + if ``pyproject.toml`` is missing or ``build-system`` is undefined. + """ + return set(self._build_system['requires']) + + def get_requires_for_build( + self, + distribution: Distribution, + config_settings: ConfigSettings | None = None, + ) -> set[str]: + """ + Return the dependencies defined by the backend in addition to + :attr:`build_system_requires` for a given distribution. + + :param distribution: Distribution to get the dependencies of + (``sdist`` or ``wheel``) + :param config_settings: Config settings for the build backend + """ + _ctx.log(f'Getting build dependencies for {distribution}...') + hook_name = f'get_requires_for_build_{distribution}' + get_requires = getattr(self._hook, hook_name) + + with self._handle_backend(hook_name): + return set(get_requires(config_settings)) + + def check_dependencies( + self, + distribution: Distribution, + config_settings: ConfigSettings | None = None, + ) -> set[tuple[str, ...]]: + """ + Return the dependencies which are not satisfied from the combined set of + :attr:`build_system_requires` and :meth:`get_requires_for_build` for a given + distribution. + + :param distribution: Distribution to check (``sdist`` or ``wheel``) + :param config_settings: Config settings for the build backend + :returns: Set of variable-length unmet dependency tuples + """ + dependencies = self.get_requires_for_build(distribution, config_settings).union(self.build_system_requires) + return {u for d in dependencies for u in check_dependency(d)} + + def prepare( + self, + distribution: Distribution, + output_directory: StrPath, + config_settings: ConfigSettings | None = None, + ) -> str | None: + """ + Prepare metadata for a distribution. + + :param distribution: Distribution to build (must be ``wheel``) + :param output_directory: Directory to put the prepared metadata in + :param config_settings: Config settings for the build backend + :returns: The full path to the prepared metadata directory + """ + _ctx.log(f'Getting metadata for {distribution}...') + try: + return self._call_backend( + f'prepare_metadata_for_build_{distribution}', + output_directory, + config_settings, + _allow_fallback=False, + ) + except BuildBackendException as exception: + if isinstance(exception.exception, pyproject_hooks.HookMissing): + return None + raise + + def build( + self, + distribution: Distribution, + output_directory: StrPath, + config_settings: ConfigSettings | None = None, + metadata_directory: str | None = None, + ) -> str: + """ + Build a distribution. + + :param distribution: Distribution to build (``sdist`` or ``wheel``) + :param output_directory: Directory to put the built distribution in + :param config_settings: Config settings for the build backend + :param metadata_directory: If provided, should be the return value of a + previous ``prepare`` call on the same ``distribution`` kind + :returns: The full path to the built distribution + """ + _ctx.log(f'Building {distribution}...') + kwargs = {} if metadata_directory is None else {'metadata_directory': metadata_directory} + return self._call_backend(f'build_{distribution}', output_directory, config_settings, **kwargs) + + def metadata_path(self, output_directory: StrPath) -> str: + """ + Generate the metadata directory of a distribution and return its path. + + If the backend does not support the ``prepare_metadata_for_build_wheel`` + hook, a wheel will be built and the metadata will be extracted from it. + + :param output_directory: Directory to put the metadata distribution in + :returns: The path of the metadata directory + """ + # prepare_metadata hook + metadata = self.prepare('wheel', output_directory) + if metadata is not None: + return metadata + + # fallback to build_wheel hook + wheel = self.build('wheel', output_directory) + match = parse_wheel_filename(os.path.basename(wheel)) + if not match: + msg = 'Invalid wheel' + raise ValueError(msg) + distinfo = f"{match['distribution']}-{match['version']}.dist-info" + member_prefix = f'{distinfo}/' + with zipfile.ZipFile(wheel) as w: + w.extractall( + output_directory, + (member for member in w.namelist() if member.startswith(member_prefix)), + ) + return os.path.join(output_directory, distinfo) + + def _call_backend( + self, hook_name: str, outdir: StrPath, config_settings: ConfigSettings | None = None, **kwargs: Any + ) -> str: + outdir = os.path.abspath(outdir) + + callback = getattr(self._hook, hook_name) + + if os.path.exists(outdir): + if not os.path.isdir(outdir): + msg = f"Build path '{outdir}' exists and is not a directory" + raise BuildException(msg) + else: + os.makedirs(outdir) + + with self._handle_backend(hook_name): + basename: str = callback(outdir, config_settings, **kwargs) + + return os.path.join(outdir, basename) + + @contextlib.contextmanager + def _handle_backend(self, hook: str) -> Iterator[None]: + try: + yield + except pyproject_hooks.BackendUnavailable as exception: + raise BuildBackendException( + exception, + f"Backend '{self._backend}' is not available.", + sys.exc_info(), + ) from None + except subprocess.CalledProcessError as exception: + raise BuildBackendException(exception, f'Backend subprocess exited when trying to invoke {hook}') from None + except Exception as exception: + raise BuildBackendException(exception, exc_info=sys.exc_info()) from None diff --git a/pipenv/vendor/build/_compat/__init__.py b/pipenv/vendor/build/_compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pipenv/vendor/build/_compat/importlib.py b/pipenv/vendor/build/_compat/importlib.py new file mode 100644 index 0000000000..758e2239ee --- /dev/null +++ b/pipenv/vendor/build/_compat/importlib.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import sys +import typing + + +if typing.TYPE_CHECKING: + import importlib_metadata as metadata +else: + if sys.version_info >= (3, 10, 2): + from importlib import metadata + else: + try: + import importlib_metadata as metadata + except ModuleNotFoundError: + # helps bootstrapping when dependencies aren't installed + from importlib import metadata + + +__all__ = [ + 'metadata', +] diff --git a/pipenv/vendor/build/_compat/tarfile.py b/pipenv/vendor/build/_compat/tarfile.py new file mode 100644 index 0000000000..2984a49790 --- /dev/null +++ b/pipenv/vendor/build/_compat/tarfile.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import sys +import tarfile +import typing + + +if typing.TYPE_CHECKING: + TarFile = tarfile.TarFile + +else: + # Per https://peps.python.org/pep-0706/, the "data" filter will become + # the default in Python 3.14. The first series of releases with the filter + # had a broken filter that could not process symlinks correctly. + if ( + (3, 8, 18) <= sys.version_info < (3, 9) + or (3, 9, 18) <= sys.version_info < (3, 10) + or (3, 10, 13) <= sys.version_info < (3, 11) + or (3, 11, 5) <= sys.version_info < (3, 12) + or (3, 12) <= sys.version_info < (3, 14) + ): + + class TarFile(tarfile.TarFile): + extraction_filter = staticmethod(tarfile.data_filter) + + else: + TarFile = tarfile.TarFile + + +__all__ = [ + 'TarFile', +] diff --git a/pipenv/vendor/build/_compat/tomllib.py b/pipenv/vendor/build/_compat/tomllib.py new file mode 100644 index 0000000000..51191efe94 --- /dev/null +++ b/pipenv/vendor/build/_compat/tomllib.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import sys + + +if sys.version_info >= (3, 11): + from tomllib import TOMLDecodeError, load, loads +else: + from tomli import TOMLDecodeError, load, loads + + +__all__ = [ + 'TOMLDecodeError', + 'load', + 'loads', +] diff --git a/pipenv/vendor/build/_ctx.py b/pipenv/vendor/build/_ctx.py new file mode 100644 index 0000000000..e65b24ea2e --- /dev/null +++ b/pipenv/vendor/build/_ctx.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import contextvars +import logging +import subprocess +import typing + +from collections.abc import Mapping, Sequence +from functools import partial + +from ._types import StrPath + + +class _Logger(typing.Protocol): # pragma: no cover + def __call__(self, message: str, *, origin: tuple[str, ...] | None = None) -> None: ... + + +_package_name = __spec__.parent # type: ignore[name-defined] +_default_logger = logging.getLogger(_package_name) + + +def _log_default(message: str, *, origin: tuple[str, ...] | None = None) -> None: + if origin is None: + _default_logger.log(logging.INFO, message, stacklevel=2) + + +LOGGER = contextvars.ContextVar('LOGGER', default=_log_default) +VERBOSITY = contextvars.ContextVar('VERBOSITY', default=0) + + +def log_subprocess_error(error: subprocess.CalledProcessError) -> None: + log = LOGGER.get() + + log(subprocess.list2cmdline(error.cmd), origin=('subprocess', 'cmd')) + + for stream_name in ('stdout', 'stderr'): + stream = getattr(error, stream_name) + if stream: + log(stream.decode() if isinstance(stream, bytes) else stream, origin=('subprocess', stream_name)) + + +def run_subprocess(cmd: Sequence[StrPath], env: Mapping[str, str] | None = None) -> None: + verbosity = VERBOSITY.get() + + if verbosity: + import concurrent.futures + + log = LOGGER.get() + + def log_stream(stream_name: str, stream: typing.IO[str]) -> None: + for line in stream: + log(line, origin=('subprocess', stream_name)) + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor, subprocess.Popen( + cmd, encoding='utf-8', env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) as process: + log(subprocess.list2cmdline(cmd), origin=('subprocess', 'cmd')) + + # Logging in sub-thread to more-or-less ensure order of stdout and stderr whilst also + # being able to distinguish between the two. + concurrent.futures.wait( + [executor.submit(partial(log_stream, n, getattr(process, n))) for n in ('stdout', 'stderr')] + ) + + code = process.wait() + if code: + raise subprocess.CalledProcessError(code, process.args) + + else: + try: + subprocess.run(cmd, capture_output=True, check=True, env=env) + except subprocess.CalledProcessError as error: + log_subprocess_error(error) + raise + + +if typing.TYPE_CHECKING: + log: _Logger + verbosity: bool + +else: + + def __getattr__(name): + if name == 'log': + return LOGGER.get() + elif name == 'verbosity': + return VERBOSITY.get() + raise AttributeError(name) # pragma: no cover + + +__all__ = [ + 'log_subprocess_error', + 'log', + 'run_subprocess', + 'LOGGER', + 'verbosity', + 'VERBOSITY', +] diff --git a/pipenv/vendor/build/_exceptions.py b/pipenv/vendor/build/_exceptions.py new file mode 100644 index 0000000000..ff14b639a7 --- /dev/null +++ b/pipenv/vendor/build/_exceptions.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import subprocess +import types + + +class BuildException(Exception): + """ + Exception raised by :class:`build.ProjectBuilder`. + """ + + +class BuildBackendException(Exception): + """ + Exception raised when a backend operation fails. + """ + + def __init__( + self, + exception: Exception, + description: str | None = None, + exc_info: tuple[type[BaseException], BaseException, types.TracebackType] | tuple[None, None, None] = ( + None, + None, + None, + ), + ) -> None: + super().__init__() + self.exception = exception + self.exc_info = exc_info + self._description = description + + def __str__(self) -> str: + if self._description: + return self._description + return f'Backend operation failed: {self.exception!r}' + + +class BuildSystemTableValidationError(BuildException): + """ + Exception raised when the ``[build-system]`` table in pyproject.toml is invalid. + """ + + def __str__(self) -> str: + return f'Failed to validate `build-system` in pyproject.toml: {self.args[0]}' + + +class FailedProcessError(Exception): + """ + Exception raised when a setup or preparation operation fails. + """ + + def __init__(self, exception: subprocess.CalledProcessError, description: str) -> None: + super().__init__() + self.exception = exception + self._description = description + + def __str__(self) -> str: + return self._description + + +class TypoWarning(Warning): + """ + Warning raised when a possible typo is found. + """ diff --git a/pipenv/vendor/build/_types.py b/pipenv/vendor/build/_types.py new file mode 100644 index 0000000000..8468a123da --- /dev/null +++ b/pipenv/vendor/build/_types.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import os +import sys +import typing + + +__all__ = ['ConfigSettings', 'Distribution', 'StrPath', 'SubprocessRunner'] + +ConfigSettings = typing.Mapping[str, typing.Union[str, typing.Sequence[str]]] +Distribution = typing.Literal['sdist', 'wheel', 'editable'] + +if typing.TYPE_CHECKING or sys.version_info > (3, 9): + StrPath = typing.Union[str, os.PathLike[str]] +else: + StrPath = typing.Union[str, os.PathLike] + +if typing.TYPE_CHECKING: + from pipenv.vendor.pyproject_hooks import SubprocessRunner +else: + SubprocessRunner = typing.Callable[ + [typing.Sequence[str], typing.Optional[str], typing.Optional[typing.Mapping[str, str]]], None + ] diff --git a/pipenv/vendor/build/_util.py b/pipenv/vendor/build/_util.py new file mode 100644 index 0000000000..a57c41eb88 --- /dev/null +++ b/pipenv/vendor/build/_util.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import re + +from collections.abc import Iterator, Set + + +_WHEEL_FILENAME_REGEX = re.compile( + r'(?P.+)-(?P.+)' + r'(-(?P.+))?-(?P.+)' + r'-(?P.+)-(?P.+)\.whl' +) + + +def check_dependency( + req_string: str, ancestral_req_strings: tuple[str, ...] = (), parent_extras: Set[str] = frozenset() +) -> Iterator[tuple[str, ...]]: + """ + Verify that a dependency and all of its dependencies are met. + + :param req_string: Requirement string + :param parent_extras: Extras (eg. "test" in myproject[test]) + :yields: Unmet dependencies + """ + import packaging.requirements + + from ._compat import importlib + + req = packaging.requirements.Requirement(req_string) + normalised_req_string = str(req) + + # ``Requirement`` doesn't implement ``__eq__`` so we cannot compare reqs for + # equality directly but the string representation is stable. + if normalised_req_string in ancestral_req_strings: + # cyclical dependency, already checked. + return + + if req.marker: + extras = frozenset(('',)).union(parent_extras) + # a requirement can have multiple extras but ``evaluate`` can + # only check one at a time. + if all(not req.marker.evaluate(environment={'extra': e}) for e in extras): + # if the marker conditions are not met, we pretend that the + # dependency is satisfied. + return + + try: + dist = importlib.metadata.distribution(req.name) + except importlib.metadata.PackageNotFoundError: + # dependency is not installed in the environment. + yield (*ancestral_req_strings, normalised_req_string) + else: + if req.specifier and not req.specifier.contains(dist.version, prereleases=True): + # the installed version is incompatible. + yield (*ancestral_req_strings, normalised_req_string) + elif dist.requires: + for other_req_string in dist.requires: + # yields transitive dependencies that are not satisfied. + yield from check_dependency(other_req_string, (*ancestral_req_strings, normalised_req_string), req.extras) + + +def parse_wheel_filename(filename: str) -> re.Match[str] | None: + return _WHEEL_FILENAME_REGEX.match(filename) diff --git a/pipenv/vendor/build/env.py b/pipenv/vendor/build/env.py new file mode 100644 index 0000000000..c0302ccae8 --- /dev/null +++ b/pipenv/vendor/build/env.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +import abc +import functools +import importlib.util +import os +import platform +import shutil +import subprocess +import sys +import sysconfig +import tempfile +import typing + +from collections.abc import Collection, Mapping + +from . import _ctx +from ._ctx import run_subprocess +from ._exceptions import FailedProcessError +from ._util import check_dependency + + +Installer = typing.Literal['pip', 'uv'] + +INSTALLERS = typing.get_args(Installer) + + +class IsolatedEnv(typing.Protocol): + """Isolated build environment ABC.""" + + @property + @abc.abstractmethod + def python_executable(self) -> str: + """The Python executable of the isolated environment.""" + + @abc.abstractmethod + def make_extra_environ(self) -> Mapping[str, str] | None: + """Generate additional env vars specific to the isolated environment.""" + + +def _has_dependency(name: str, minimum_version_str: str | None = None, /, **distargs: object) -> bool | None: + """ + Given a path, see if a package is present and return True if the version is + sufficient for build, False if it is not, None if the package is missing. + """ + from pipenv.vendor.packaging.version import Version + + from ._compat import importlib + + try: + distribution = next(iter(importlib.metadata.distributions(name=name, **distargs))) + except StopIteration: + return None + + if minimum_version_str is None: + return True + + return Version(distribution.version) >= Version(minimum_version_str) + + +class DefaultIsolatedEnv(IsolatedEnv): + """ + Isolated environment which supports several different underlying implementations. + """ + + def __init__( + self, + *, + installer: Installer = 'pip', + ) -> None: + self.installer: Installer = installer + + def __enter__(self) -> DefaultIsolatedEnv: + try: + path = tempfile.mkdtemp(prefix='build-env-') + # Call ``realpath`` to prevent spurious warning from being emitted + # that the venv location has changed on Windows for the venv impl. + # The username is DOS-encoded in the output of tempfile - the location is the same + # but the representation of it is different, which confuses venv. + # Ref: https://bugs.python.org/issue46171 + path = os.path.realpath(path) + self._path = path + + self._env_backend: _EnvBackend + + # uv is opt-in only. + if self.installer == 'uv': + self._env_backend = _UvBackend() + else: + self._env_backend = _PipBackend() + + _ctx.log(f'Creating isolated environment: {self._env_backend.display_name}...') + self._env_backend.create(self._path) + + except Exception: # cleanup folder if creation fails + self.__exit__(*sys.exc_info()) + raise + + return self + + def __exit__(self, *args: object) -> None: + if os.path.exists(self._path): # in case the user already deleted skip remove + shutil.rmtree(self._path) + + @property + def path(self) -> str: + """The location of the isolated build environment.""" + return self._path + + @property + def python_executable(self) -> str: + """The python executable of the isolated build environment.""" + return self._env_backend.python_executable + + def make_extra_environ(self) -> dict[str, str]: + path = os.environ.get('PATH') + return { + 'PATH': os.pathsep.join([self._env_backend.scripts_dir, path]) + if path is not None + else self._env_backend.scripts_dir + } + + def install(self, requirements: Collection[str]) -> None: + """ + Install packages from PEP 508 requirements in the isolated build environment. + + :param requirements: PEP 508 requirement specification to install + + :note: Passing non-PEP 508 strings will result in undefined behavior, you *should not* rely on it. It is + merely an implementation detail, it may change any time without warning. + """ + if not requirements: + return + + _ctx.log('Installing packages in isolated environment:\n' + '\n'.join(f'- {r}' for r in sorted(requirements))) + self._env_backend.install_requirements(requirements) + + +class _EnvBackend(typing.Protocol): # pragma: no cover + python_executable: str + scripts_dir: str + + def create(self, path: str) -> None: ... + + def install_requirements(self, requirements: Collection[str]) -> None: ... + + @property + def display_name(self) -> str: ... + + +class _PipBackend(_EnvBackend): + def __init__(self) -> None: + self._create_with_virtualenv = not self._has_valid_outer_pip and self._has_virtualenv + + @functools.cached_property + def _has_valid_outer_pip(self) -> bool | None: + """ + This checks for a valid global pip. Returns None if pip is missing, False + if pip is too old, and True if it can be used. + """ + # Version to have added the `--python` option. + return _has_dependency('pip', '22.3') + + @functools.cached_property + def _has_virtualenv(self) -> bool: + """ + virtualenv might be incompatible if it was installed separately + from pipenv.vendor.build. This verifies that virtualenv and all of its + dependencies are installed as required by build. + """ + from pipenv.vendor.packaging.requirements import Requirement + + name = 'virtualenv' + + return importlib.util.find_spec(name) is not None and not any( + Requirement(d[1]).name == name for d in check_dependency(f'build[{name}]') if len(d) > 1 + ) + + @staticmethod + def _get_minimum_pip_version_str() -> str: + if platform.system() == 'Darwin': + release, _, machine = platform.mac_ver() + if int(release[: release.find('.')]) >= 11: + # macOS 11+ name scheme change requires 20.3. Intel macOS 11.0 can be + # told to report 10.16 for backwards compatibility; but that also fixes + # earlier versions of pip so this is only needed for 11+. + is_apple_silicon_python = machine != 'x86_64' + return '21.0.1' if is_apple_silicon_python else '20.3.0' + + # PEP-517 and manylinux1 was first implemented in 19.1 + return '19.1.0' + + def create(self, path: str) -> None: + if self._create_with_virtualenv: + import virtualenv + + result = virtualenv.cli_run( + [ + path, + '--activators', + '', + '--no-setuptools', + '--no-wheel', + ], + setup_logging=False, + ) + + # The creator attributes are `pathlib.Path`s. + self.python_executable = str(result.creator.exe) + self.scripts_dir = str(result.creator.script_dir) + + else: + import venv + + with_pip = not self._has_valid_outer_pip + + try: + venv.EnvBuilder(symlinks=_fs_supports_symlink(), with_pip=with_pip).create(path) + except subprocess.CalledProcessError as exc: + _ctx.log_subprocess_error(exc) + raise FailedProcessError(exc, 'Failed to create venv. Maybe try installing virtualenv.') from None + + self.python_executable, self.scripts_dir, purelib = _find_executable_and_scripts(path) + + if with_pip: + minimum_pip_version_str = self._get_minimum_pip_version_str() + if not _has_dependency( + 'pip', + minimum_pip_version_str, + path=[purelib], + ): + run_subprocess([self.python_executable, '-Im', 'pip', 'install', f'pip>={minimum_pip_version_str}']) + + # Uninstall setuptools from the build env to prevent depending on it implicitly. + # Pythons 3.12 and up do not install setuptools, check if it exists first. + if _has_dependency( + 'setuptools', + path=[purelib], + ): + run_subprocess([self.python_executable, '-Im', 'pip', 'uninstall', '-y', 'setuptools']) + + def install_requirements(self, requirements: Collection[str]) -> None: + # pip does not honour environment markers in command line arguments + # but it does from requirement files. + with tempfile.NamedTemporaryFile('w', prefix='build-reqs-', suffix='.txt', delete=False, encoding='utf-8') as req_file: + req_file.write(os.linesep.join(requirements)) + + try: + if self._has_valid_outer_pip: + cmd = [sys.executable, '-m', 'pip', '--python', self.python_executable] + else: + cmd = [self.python_executable, '-Im', 'pip'] + + if _ctx.verbosity > 1: + cmd += [f'-{"v" * (_ctx.verbosity - 1)}'] + + cmd += [ + 'install', + '--use-pep517', + '--no-warn-script-location', + '--no-compile', + '-r', + os.path.abspath(req_file.name), + ] + run_subprocess(cmd) + + finally: + os.unlink(req_file.name) + + @property + def display_name(self) -> str: + return 'virtualenv+pip' if self._create_with_virtualenv else 'venv+pip' + + +class _UvBackend(_EnvBackend): + def create(self, path: str) -> None: + import venv + + self._env_path = path + + try: + import uv + + self._uv_bin = uv.find_uv_bin() + except (ModuleNotFoundError, FileNotFoundError): + uv_bin = shutil.which('uv') + if uv_bin is None: + msg = 'uv executable not found' + raise RuntimeError(msg) from None + + _ctx.log(f'Using external uv from {uv_bin}') + self._uv_bin = uv_bin + + venv.EnvBuilder(symlinks=_fs_supports_symlink(), with_pip=False).create(self._env_path) + self.python_executable, self.scripts_dir, _ = _find_executable_and_scripts(self._env_path) + + def install_requirements(self, requirements: Collection[str]) -> None: + cmd = [self._uv_bin, 'pip'] + if _ctx.verbosity > 1: + cmd += [f'-{"v" * min(2, _ctx.verbosity - 1)}'] + run_subprocess([*cmd, 'install', *requirements], env={**os.environ, 'VIRTUAL_ENV': self._env_path}) + + @property + def display_name(self) -> str: + return 'venv+uv' + + +@functools.lru_cache(maxsize=None) +def _fs_supports_symlink() -> bool: + """Return True if symlinks are supported""" + # Using definition used by venv.main() + if os.name != 'nt': + return True + + # Windows may support symlinks (setting in Windows 10) + with tempfile.NamedTemporaryFile(prefix='build-symlink-') as tmp_file: + dest = f'{tmp_file}-b' + try: + os.symlink(tmp_file.name, dest) + os.unlink(dest) + except (OSError, NotImplementedError, AttributeError): + return False + return True + + +def _find_executable_and_scripts(path: str) -> tuple[str, str, str]: + """ + Detect the Python executable and script folder of a virtual environment. + + :param path: The location of the virtual environment + :return: The Python executable, script folder, and purelib folder + """ + config_vars = sysconfig.get_config_vars().copy() # globally cached, copy before altering it + config_vars['base'] = path + scheme_names = sysconfig.get_scheme_names() + if 'venv' in scheme_names: + # Python distributors with custom default installation scheme can set a + # scheme that can't be used to expand the paths in a venv. + # This can happen if build itself is not installed in a venv. + # The distributors are encouraged to set a "venv" scheme to be used for this. + # See https://bugs.python.org/issue45413 + # and https://github.com/pypa/virtualenv/issues/2208 + paths = sysconfig.get_paths(scheme='venv', vars=config_vars) + elif 'posix_local' in scheme_names: + # The Python that ships on Debian/Ubuntu varies the default scheme to + # install to /usr/local + # But it does not (yet) set the "venv" scheme. + # If we're the Debian "posix_local" scheme is available, but "venv" + # is not, we use "posix_prefix" instead which is venv-compatible there. + paths = sysconfig.get_paths(scheme='posix_prefix', vars=config_vars) + elif 'osx_framework_library' in scheme_names: + # The Python that ships with the macOS developer tools varies the + # default scheme depending on whether the ``sys.prefix`` is part of a framework. + # But it does not (yet) set the "venv" scheme. + # If the Apple-custom "osx_framework_library" scheme is available but "venv" + # is not, we use "posix_prefix" instead which is venv-compatible there. + paths = sysconfig.get_paths(scheme='posix_prefix', vars=config_vars) + else: + paths = sysconfig.get_paths(vars=config_vars) + + executable = os.path.join(paths['scripts'], 'python.exe' if os.name == 'nt' else 'python') + if not os.path.exists(executable): + msg = f'Virtual environment creation failed, executable {executable} missing' + raise RuntimeError(msg) + + return executable, paths['scripts'], paths['purelib'] + + +__all__ = [ + 'IsolatedEnv', + 'DefaultIsolatedEnv', +] diff --git a/pipenv/vendor/build/py.typed b/pipenv/vendor/build/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pipenv/vendor/build/util.py b/pipenv/vendor/build/util.py new file mode 100644 index 0000000000..02280cce9c --- /dev/null +++ b/pipenv/vendor/build/util.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import pathlib +import tempfile + +import pipenv.vendor.pyproject_hooks as pyproject_hooks + +from . import ProjectBuilder +from ._compat import importlib +from ._types import StrPath, SubprocessRunner +from .env import DefaultIsolatedEnv + + +def _project_wheel_metadata(builder: ProjectBuilder) -> importlib.metadata.PackageMetadata: + with tempfile.TemporaryDirectory() as tmpdir: + path = pathlib.Path(builder.metadata_path(tmpdir)) + return importlib.metadata.PathDistribution(path).metadata + + +def project_wheel_metadata( + source_dir: StrPath, + isolated: bool = True, + *, + runner: SubprocessRunner = pyproject_hooks.quiet_subprocess_runner, +) -> importlib.metadata.PackageMetadata: + """ + Return the wheel metadata for a project. + + Uses the ``prepare_metadata_for_build_wheel`` hook if available, + otherwise ``build_wheel``. + + :param source_dir: Project source directory + :param isolated: Whether or not to run invoke the backend in the current + environment or to create an isolated one and invoke it + there. + :param runner: An alternative runner for backend subprocesses + """ + + if isolated: + with DefaultIsolatedEnv() as env: + builder = ProjectBuilder.from_isolated_env( + env, + source_dir, + runner=runner, + ) + env.install(builder.build_system_requires) + env.install(builder.get_requires_for_build('wheel')) + return _project_wheel_metadata(builder) + else: + builder = ProjectBuilder( + source_dir, + runner=runner, + ) + return _project_wheel_metadata(builder) + + +__all__ = [ + 'project_wheel_metadata', +] diff --git a/pipenv/vendor/pyproject_hooks/LICENSE b/pipenv/vendor/pyproject_hooks/LICENSE new file mode 100644 index 0000000000..b0ae9dbc26 --- /dev/null +++ b/pipenv/vendor/pyproject_hooks/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Thomas Kluyver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/vendor/pyproject_hooks/__init__.py b/pipenv/vendor/pyproject_hooks/__init__.py new file mode 100644 index 0000000000..746b89f7e7 --- /dev/null +++ b/pipenv/vendor/pyproject_hooks/__init__.py @@ -0,0 +1,31 @@ +"""Wrappers to call pyproject.toml-based build backend hooks. +""" + +from typing import TYPE_CHECKING + +from ._impl import ( + BackendUnavailable, + BuildBackendHookCaller, + HookMissing, + UnsupportedOperation, + default_subprocess_runner, + quiet_subprocess_runner, +) + +__version__ = "1.2.0" +__all__ = [ + "BackendUnavailable", + "BackendInvalid", + "HookMissing", + "UnsupportedOperation", + "default_subprocess_runner", + "quiet_subprocess_runner", + "BuildBackendHookCaller", +] + +BackendInvalid = BackendUnavailable # Deprecated alias, previously a separate exception + +if TYPE_CHECKING: + from ._impl import SubprocessRunner + + __all__ += ["SubprocessRunner"] diff --git a/pipenv/vendor/pyproject_hooks/_impl.py b/pipenv/vendor/pyproject_hooks/_impl.py new file mode 100644 index 0000000000..d1e9d7bb8c --- /dev/null +++ b/pipenv/vendor/pyproject_hooks/_impl.py @@ -0,0 +1,410 @@ +import json +import os +import sys +import tempfile +from contextlib import contextmanager +from os.path import abspath +from os.path import join as pjoin +from subprocess import STDOUT, check_call, check_output +from typing import TYPE_CHECKING, Any, Iterator, Mapping, Optional, Sequence + +from ._in_process import _in_proc_script_path + +if TYPE_CHECKING: + from typing import Protocol + + class SubprocessRunner(Protocol): + """A protocol for the subprocess runner.""" + + def __call__( + self, + cmd: Sequence[str], + cwd: Optional[str] = None, + extra_environ: Optional[Mapping[str, str]] = None, + ) -> None: + ... + + +def write_json(obj: Mapping[str, Any], path: str, **kwargs) -> None: + with open(path, "w", encoding="utf-8") as f: + json.dump(obj, f, **kwargs) + + +def read_json(path: str) -> Mapping[str, Any]: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +class BackendUnavailable(Exception): + """Will be raised if the backend cannot be imported in the hook process.""" + + def __init__( + self, + traceback: str, + message: Optional[str] = None, + backend_name: Optional[str] = None, + backend_path: Optional[Sequence[str]] = None, + ) -> None: + # Preserving arg order for the sake of API backward compatibility. + self.backend_name = backend_name + self.backend_path = backend_path + self.traceback = traceback + super().__init__(message or "Error while importing backend") + + +class HookMissing(Exception): + """Will be raised on missing hooks (if a fallback can't be used).""" + + def __init__(self, hook_name: str) -> None: + super().__init__(hook_name) + self.hook_name = hook_name + + +class UnsupportedOperation(Exception): + """May be raised by build_sdist if the backend indicates that it can't.""" + + def __init__(self, traceback: str) -> None: + self.traceback = traceback + + +def default_subprocess_runner( + cmd: Sequence[str], + cwd: Optional[str] = None, + extra_environ: Optional[Mapping[str, str]] = None, +) -> None: + """The default method of calling the wrapper subprocess. + + This uses :func:`subprocess.check_call` under the hood. + """ + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_call(cmd, cwd=cwd, env=env) + + +def quiet_subprocess_runner( + cmd: Sequence[str], + cwd: Optional[str] = None, + extra_environ: Optional[Mapping[str, str]] = None, +) -> None: + """Call the subprocess while suppressing output. + + This uses :func:`subprocess.check_output` under the hood. + """ + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_output(cmd, cwd=cwd, env=env, stderr=STDOUT) + + +def norm_and_check(source_tree: str, requested: str) -> str: + """Normalise and check a backend path. + + Ensure that the requested backend path is specified as a relative path, + and resolves to a location under the given source tree. + + Return an absolute version of the requested path. + """ + if os.path.isabs(requested): + raise ValueError("paths must be relative") + + abs_source = os.path.abspath(source_tree) + abs_requested = os.path.normpath(os.path.join(abs_source, requested)) + # We have to use commonprefix for Python 2.7 compatibility. So we + # normalise case to avoid problems because commonprefix is a character + # based comparison :-( + norm_source = os.path.normcase(abs_source) + norm_requested = os.path.normcase(abs_requested) + if os.path.commonprefix([norm_source, norm_requested]) != norm_source: + raise ValueError("paths must be inside source tree") + + return abs_requested + + +class BuildBackendHookCaller: + """A wrapper to call the build backend hooks for a source directory.""" + + def __init__( + self, + source_dir: str, + build_backend: str, + backend_path: Optional[Sequence[str]] = None, + runner: Optional["SubprocessRunner"] = None, + python_executable: Optional[str] = None, + ) -> None: + """ + :param source_dir: The source directory to invoke the build backend for + :param build_backend: The build backend spec + :param backend_path: Additional path entries for the build backend spec + :param runner: The :ref:`subprocess runner ` to use + :param python_executable: + The Python executable used to invoke the build backend + """ + if runner is None: + runner = default_subprocess_runner + + self.source_dir = abspath(source_dir) + self.build_backend = build_backend + if backend_path: + backend_path = [norm_and_check(self.source_dir, p) for p in backend_path] + self.backend_path = backend_path + self._subprocess_runner = runner + if not python_executable: + python_executable = sys.executable + self.python_executable = python_executable + + @contextmanager + def subprocess_runner(self, runner: "SubprocessRunner") -> Iterator[None]: + """A context manager for temporarily overriding the default + :ref:`subprocess runner `. + + :param runner: The new subprocess runner to use within the context. + + .. code-block:: python + + hook_caller = BuildBackendHookCaller(...) + with hook_caller.subprocess_runner(quiet_subprocess_runner): + ... + """ + prev = self._subprocess_runner + self._subprocess_runner = runner + try: + yield + finally: + self._subprocess_runner = prev + + def _supported_features(self) -> Sequence[str]: + """Return the list of optional features supported by the backend.""" + return self._call_hook("_supported_features", {}) + + def get_requires_for_build_wheel( + self, + config_settings: Optional[Mapping[str, Any]] = None, + ) -> Sequence[str]: + """Get additional dependencies required for building a wheel. + + :param config_settings: The configuration settings for the build backend + :returns: A list of :pep:`dependency specifiers <508>`. + + .. admonition:: Fallback + + If the build backend does not defined a hook with this name, an + empty list will be returned. + """ + return self._call_hook( + "get_requires_for_build_wheel", {"config_settings": config_settings} + ) + + def prepare_metadata_for_build_wheel( + self, + metadata_directory: str, + config_settings: Optional[Mapping[str, Any]] = None, + _allow_fallback: bool = True, + ) -> str: + """Prepare a ``*.dist-info`` folder with metadata for this project. + + :param metadata_directory: The directory to write the metadata to + :param config_settings: The configuration settings for the build backend + :param _allow_fallback: + Whether to allow the fallback to building a wheel and extracting + the metadata from it. Should be passed as a keyword argument only. + + :returns: Name of the newly created subfolder within + ``metadata_directory``, containing the metadata. + + .. admonition:: Fallback + + If the build backend does not define a hook with this name and + ``_allow_fallback`` is truthy, the backend will be asked to build a + wheel via the ``build_wheel`` hook and the dist-info extracted from + that will be returned. + """ + return self._call_hook( + "prepare_metadata_for_build_wheel", + { + "metadata_directory": abspath(metadata_directory), + "config_settings": config_settings, + "_allow_fallback": _allow_fallback, + }, + ) + + def build_wheel( + self, + wheel_directory: str, + config_settings: Optional[Mapping[str, Any]] = None, + metadata_directory: Optional[str] = None, + ) -> str: + """Build a wheel from this project. + + :param wheel_directory: The directory to write the wheel to + :param config_settings: The configuration settings for the build backend + :param metadata_directory: The directory to reuse existing metadata from + :returns: + The name of the newly created wheel within ``wheel_directory``. + + .. admonition:: Interaction with fallback + + If the ``build_wheel`` hook was called in the fallback for + :meth:`prepare_metadata_for_build_wheel`, the build backend would + not be invoked. Instead, the previously built wheel will be copied + to ``wheel_directory`` and the name of that file will be returned. + """ + if metadata_directory is not None: + metadata_directory = abspath(metadata_directory) + return self._call_hook( + "build_wheel", + { + "wheel_directory": abspath(wheel_directory), + "config_settings": config_settings, + "metadata_directory": metadata_directory, + }, + ) + + def get_requires_for_build_editable( + self, + config_settings: Optional[Mapping[str, Any]] = None, + ) -> Sequence[str]: + """Get additional dependencies required for building an editable wheel. + + :param config_settings: The configuration settings for the build backend + :returns: A list of :pep:`dependency specifiers <508>`. + + .. admonition:: Fallback + + If the build backend does not defined a hook with this name, an + empty list will be returned. + """ + return self._call_hook( + "get_requires_for_build_editable", {"config_settings": config_settings} + ) + + def prepare_metadata_for_build_editable( + self, + metadata_directory: str, + config_settings: Optional[Mapping[str, Any]] = None, + _allow_fallback: bool = True, + ) -> Optional[str]: + """Prepare a ``*.dist-info`` folder with metadata for this project. + + :param metadata_directory: The directory to write the metadata to + :param config_settings: The configuration settings for the build backend + :param _allow_fallback: + Whether to allow the fallback to building a wheel and extracting + the metadata from it. Should be passed as a keyword argument only. + :returns: Name of the newly created subfolder within + ``metadata_directory``, containing the metadata. + + .. admonition:: Fallback + + If the build backend does not define a hook with this name and + ``_allow_fallback`` is truthy, the backend will be asked to build a + wheel via the ``build_editable`` hook and the dist-info + extracted from that will be returned. + """ + return self._call_hook( + "prepare_metadata_for_build_editable", + { + "metadata_directory": abspath(metadata_directory), + "config_settings": config_settings, + "_allow_fallback": _allow_fallback, + }, + ) + + def build_editable( + self, + wheel_directory: str, + config_settings: Optional[Mapping[str, Any]] = None, + metadata_directory: Optional[str] = None, + ) -> str: + """Build an editable wheel from this project. + + :param wheel_directory: The directory to write the wheel to + :param config_settings: The configuration settings for the build backend + :param metadata_directory: The directory to reuse existing metadata from + :returns: + The name of the newly created wheel within ``wheel_directory``. + + .. admonition:: Interaction with fallback + + If the ``build_editable`` hook was called in the fallback for + :meth:`prepare_metadata_for_build_editable`, the build backend + would not be invoked. Instead, the previously built wheel will be + copied to ``wheel_directory`` and the name of that file will be + returned. + """ + if metadata_directory is not None: + metadata_directory = abspath(metadata_directory) + return self._call_hook( + "build_editable", + { + "wheel_directory": abspath(wheel_directory), + "config_settings": config_settings, + "metadata_directory": metadata_directory, + }, + ) + + def get_requires_for_build_sdist( + self, + config_settings: Optional[Mapping[str, Any]] = None, + ) -> Sequence[str]: + """Get additional dependencies required for building an sdist. + + :returns: A list of :pep:`dependency specifiers <508>`. + """ + return self._call_hook( + "get_requires_for_build_sdist", {"config_settings": config_settings} + ) + + def build_sdist( + self, + sdist_directory: str, + config_settings: Optional[Mapping[str, Any]] = None, + ) -> str: + """Build an sdist from this project. + + :returns: + The name of the newly created sdist within ``wheel_directory``. + """ + return self._call_hook( + "build_sdist", + { + "sdist_directory": abspath(sdist_directory), + "config_settings": config_settings, + }, + ) + + def _call_hook(self, hook_name: str, kwargs: Mapping[str, Any]) -> Any: + extra_environ = {"_PYPROJECT_HOOKS_BUILD_BACKEND": self.build_backend} + + if self.backend_path: + backend_path = os.pathsep.join(self.backend_path) + extra_environ["_PYPROJECT_HOOKS_BACKEND_PATH"] = backend_path + + with tempfile.TemporaryDirectory() as td: + hook_input = {"kwargs": kwargs} + write_json(hook_input, pjoin(td, "input.json"), indent=2) + + # Run the hook in a subprocess + with _in_proc_script_path() as script: + python = self.python_executable + self._subprocess_runner( + [python, abspath(str(script)), hook_name, td], + cwd=self.source_dir, + extra_environ=extra_environ, + ) + + data = read_json(pjoin(td, "output.json")) + if data.get("unsupported"): + raise UnsupportedOperation(data.get("traceback", "")) + if data.get("no_backend"): + raise BackendUnavailable( + data.get("traceback", ""), + message=data.get("backend_error", ""), + backend_name=self.build_backend, + backend_path=self.backend_path, + ) + if data.get("hook_missing"): + raise HookMissing(data.get("missing_hook_name") or hook_name) + return data["return_val"] diff --git a/pipenv/vendor/pyproject_hooks/_in_process/__init__.py b/pipenv/vendor/pyproject_hooks/_in_process/__init__.py new file mode 100644 index 0000000000..906d0ba20e --- /dev/null +++ b/pipenv/vendor/pyproject_hooks/_in_process/__init__.py @@ -0,0 +1,21 @@ +"""This is a subpackage because the directory is on sys.path for _in_process.py + +The subpackage should stay as empty as possible to avoid shadowing modules that +the backend might import. +""" + +import importlib.resources as resources + +try: + resources.files +except AttributeError: + # Python 3.8 compatibility + def _in_proc_script_path(): + return resources.path(__package__, "_in_process.py") + +else: + + def _in_proc_script_path(): + return resources.as_file( + resources.files(__package__).joinpath("_in_process.py") + ) diff --git a/pipenv/vendor/pyproject_hooks/_in_process/_in_process.py b/pipenv/vendor/pyproject_hooks/_in_process/_in_process.py new file mode 100644 index 0000000000..d689bab721 --- /dev/null +++ b/pipenv/vendor/pyproject_hooks/_in_process/_in_process.py @@ -0,0 +1,389 @@ +"""This is invoked in a subprocess to call the build backend hooks. + +It expects: +- Command line args: hook_name, control_dir +- Environment variables: + _PYPROJECT_HOOKS_BUILD_BACKEND=entry.point:spec + _PYPROJECT_HOOKS_BACKEND_PATH=paths (separated with os.pathsep) +- control_dir/input.json: + - {"kwargs": {...}} + +Results: +- control_dir/output.json + - {"return_val": ...} +""" +import json +import os +import os.path +import re +import shutil +import sys +import traceback +from glob import glob +from importlib import import_module +from importlib.machinery import PathFinder +from os.path import join as pjoin + +# This file is run as a script, and `import wrappers` is not zip-safe, so we +# include write_json() and read_json() from wrappers.py. + + +def write_json(obj, path, **kwargs): + with open(path, "w", encoding="utf-8") as f: + json.dump(obj, f, **kwargs) + + +def read_json(path): + with open(path, encoding="utf-8") as f: + return json.load(f) + + +class BackendUnavailable(Exception): + """Raised if we cannot import the backend""" + + def __init__(self, message, traceback=None): + super().__init__(message) + self.message = message + self.traceback = traceback + + +class HookMissing(Exception): + """Raised if a hook is missing and we are not executing the fallback""" + + def __init__(self, hook_name=None): + super().__init__(hook_name) + self.hook_name = hook_name + + +def _build_backend(): + """Find and load the build backend""" + backend_path = os.environ.get("_PYPROJECT_HOOKS_BACKEND_PATH") + ep = os.environ["_PYPROJECT_HOOKS_BUILD_BACKEND"] + mod_path, _, obj_path = ep.partition(":") + + if backend_path: + # Ensure in-tree backend directories have the highest priority when importing. + extra_pathitems = backend_path.split(os.pathsep) + sys.meta_path.insert(0, _BackendPathFinder(extra_pathitems, mod_path)) + + try: + obj = import_module(mod_path) + except ImportError: + msg = f"Cannot import {mod_path!r}" + raise BackendUnavailable(msg, traceback.format_exc()) + + if obj_path: + for path_part in obj_path.split("."): + obj = getattr(obj, path_part) + return obj + + +class _BackendPathFinder: + """Implements the MetaPathFinder interface to locate modules in ``backend-path``. + + Since the environment provided by the frontend can contain all sorts of + MetaPathFinders, the only way to ensure the backend is loaded from the + right place is to prepend our own. + """ + + def __init__(self, backend_path, backend_module): + self.backend_path = backend_path + self.backend_module = backend_module + self.backend_parent, _, _ = backend_module.partition(".") + + def find_spec(self, fullname, _path, _target=None): + if "." in fullname: + # Rely on importlib to find nested modules based on parent's path + return None + + # Ignore other items in _path or sys.path and use backend_path instead: + spec = PathFinder.find_spec(fullname, path=self.backend_path) + if spec is None and fullname == self.backend_parent: + # According to the spec, the backend MUST be loaded from backend-path. + # Therefore, we can halt the import machinery and raise a clean error. + msg = f"Cannot find module {self.backend_module!r} in {self.backend_path!r}" + raise BackendUnavailable(msg) + + return spec + + if sys.version_info >= (3, 8): + + def find_distributions(self, context=None): + # Delayed import: Python 3.7 does not contain importlib.metadata + from importlib.metadata import DistributionFinder, MetadataPathFinder + + context = DistributionFinder.Context(path=self.backend_path) + return MetadataPathFinder.find_distributions(context=context) + + +def _supported_features(): + """Return the list of options features supported by the backend. + + Returns a list of strings. + The only possible value is 'build_editable'. + """ + backend = _build_backend() + features = [] + if hasattr(backend, "build_editable"): + features.append("build_editable") + return features + + +def get_requires_for_build_wheel(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_wheel + except AttributeError: + return [] + else: + return hook(config_settings) + + +def get_requires_for_build_editable(config_settings): + """Invoke the optional get_requires_for_build_editable hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_editable + except AttributeError: + return [] + else: + return hook(config_settings) + + +def prepare_metadata_for_build_wheel( + metadata_directory, config_settings, _allow_fallback +): + """Invoke optional prepare_metadata_for_build_wheel + + Implements a fallback by building a wheel if the hook isn't defined, + unless _allow_fallback is False in which case HookMissing is raised. + """ + backend = _build_backend() + try: + hook = backend.prepare_metadata_for_build_wheel + except AttributeError: + if not _allow_fallback: + raise HookMissing() + else: + return hook(metadata_directory, config_settings) + # fallback to build_wheel outside the try block to avoid exception chaining + # which can be confusing to users and is not relevant + whl_basename = backend.build_wheel(metadata_directory, config_settings) + return _get_wheel_metadata_from_wheel( + whl_basename, metadata_directory, config_settings + ) + + +def prepare_metadata_for_build_editable( + metadata_directory, config_settings, _allow_fallback +): + """Invoke optional prepare_metadata_for_build_editable + + Implements a fallback by building an editable wheel if the hook isn't + defined, unless _allow_fallback is False in which case HookMissing is + raised. + """ + backend = _build_backend() + try: + hook = backend.prepare_metadata_for_build_editable + except AttributeError: + if not _allow_fallback: + raise HookMissing() + try: + build_hook = backend.build_editable + except AttributeError: + raise HookMissing(hook_name="build_editable") + else: + whl_basename = build_hook(metadata_directory, config_settings) + return _get_wheel_metadata_from_wheel( + whl_basename, metadata_directory, config_settings + ) + else: + return hook(metadata_directory, config_settings) + + +WHEEL_BUILT_MARKER = "PYPROJECT_HOOKS_ALREADY_BUILT_WHEEL" + + +def _dist_info_files(whl_zip): + """Identify the .dist-info folder inside a wheel ZipFile.""" + res = [] + for path in whl_zip.namelist(): + m = re.match(r"[^/\\]+-[^/\\]+\.dist-info/", path) + if m: + res.append(path) + if res: + return res + raise Exception("No .dist-info folder found in wheel") + + +def _get_wheel_metadata_from_wheel(whl_basename, metadata_directory, config_settings): + """Extract the metadata from a wheel. + + Fallback for when the build backend does not + define the 'get_wheel_metadata' hook. + """ + from zipfile import ZipFile + + with open(os.path.join(metadata_directory, WHEEL_BUILT_MARKER), "wb"): + pass # Touch marker file + + whl_file = os.path.join(metadata_directory, whl_basename) + with ZipFile(whl_file) as zipf: + dist_info = _dist_info_files(zipf) + zipf.extractall(path=metadata_directory, members=dist_info) + return dist_info[0].split("/")[0] + + +def _find_already_built_wheel(metadata_directory): + """Check for a wheel already built during the get_wheel_metadata hook.""" + if not metadata_directory: + return None + metadata_parent = os.path.dirname(metadata_directory) + if not os.path.isfile(pjoin(metadata_parent, WHEEL_BUILT_MARKER)): + return None + + whl_files = glob(os.path.join(metadata_parent, "*.whl")) + if not whl_files: + print("Found wheel built marker, but no .whl files") + return None + if len(whl_files) > 1: + print( + "Found multiple .whl files; unspecified behaviour. " + "Will call build_wheel." + ) + return None + + # Exactly one .whl file + return whl_files[0] + + +def build_wheel(wheel_directory, config_settings, metadata_directory=None): + """Invoke the mandatory build_wheel hook. + + If a wheel was already built in the + prepare_metadata_for_build_wheel fallback, this + will copy it rather than rebuilding the wheel. + """ + prebuilt_whl = _find_already_built_wheel(metadata_directory) + if prebuilt_whl: + shutil.copy2(prebuilt_whl, wheel_directory) + return os.path.basename(prebuilt_whl) + + return _build_backend().build_wheel( + wheel_directory, config_settings, metadata_directory + ) + + +def build_editable(wheel_directory, config_settings, metadata_directory=None): + """Invoke the optional build_editable hook. + + If a wheel was already built in the + prepare_metadata_for_build_editable fallback, this + will copy it rather than rebuilding the wheel. + """ + backend = _build_backend() + try: + hook = backend.build_editable + except AttributeError: + raise HookMissing() + else: + prebuilt_whl = _find_already_built_wheel(metadata_directory) + if prebuilt_whl: + shutil.copy2(prebuilt_whl, wheel_directory) + return os.path.basename(prebuilt_whl) + + return hook(wheel_directory, config_settings, metadata_directory) + + +def get_requires_for_build_sdist(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_sdist + except AttributeError: + return [] + else: + return hook(config_settings) + + +class _DummyException(Exception): + """Nothing should ever raise this exception""" + + +class GotUnsupportedOperation(Exception): + """For internal use when backend raises UnsupportedOperation""" + + def __init__(self, traceback): + self.traceback = traceback + + +def build_sdist(sdist_directory, config_settings): + """Invoke the mandatory build_sdist hook.""" + backend = _build_backend() + try: + return backend.build_sdist(sdist_directory, config_settings) + except getattr(backend, "UnsupportedOperation", _DummyException): + raise GotUnsupportedOperation(traceback.format_exc()) + + +HOOK_NAMES = { + "get_requires_for_build_wheel", + "prepare_metadata_for_build_wheel", + "build_wheel", + "get_requires_for_build_editable", + "prepare_metadata_for_build_editable", + "build_editable", + "get_requires_for_build_sdist", + "build_sdist", + "_supported_features", +} + + +def main(): + if len(sys.argv) < 3: + sys.exit("Needs args: hook_name, control_dir") + hook_name = sys.argv[1] + control_dir = sys.argv[2] + if hook_name not in HOOK_NAMES: + sys.exit("Unknown hook: %s" % hook_name) + + # Remove the parent directory from sys.path to avoid polluting the backend + # import namespace with this directory. + here = os.path.dirname(__file__) + if here in sys.path: + sys.path.remove(here) + + hook = globals()[hook_name] + + hook_input = read_json(pjoin(control_dir, "input.json")) + + json_out = {"unsupported": False, "return_val": None} + try: + json_out["return_val"] = hook(**hook_input["kwargs"]) + except BackendUnavailable as e: + json_out["no_backend"] = True + json_out["traceback"] = e.traceback + json_out["backend_error"] = e.message + except GotUnsupportedOperation as e: + json_out["unsupported"] = True + json_out["traceback"] = e.traceback + except HookMissing as e: + json_out["hook_missing"] = True + json_out["missing_hook_name"] = e.hook_name or hook_name + + write_json(json_out, pjoin(control_dir, "output.json"), indent=2) + + +if __name__ == "__main__": + main() diff --git a/pipenv/vendor/pyproject_hooks/py.typed b/pipenv/vendor/pyproject_hooks/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index f07df9974a..c6fc729504 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -1,8 +1,10 @@ +build==1.2.2 packaging==26.0 pexpect==4.9.0 pipdeptree==2.34.0 plette==2.2.1 ptyprocess==0.7.0 +pyproject_hooks==1.2.0 python-dotenv==1.2.2 pythonfinder==3.0.3 shellingham==1.5.4 diff --git a/tests/unit/test_candidate.py b/tests/unit/test_candidate.py index cc49018bfd..54473fa3de 100644 --- a/tests/unit/test_candidate.py +++ b/tests/unit/test_candidate.py @@ -633,3 +633,101 @@ def test_from_filename_positional_drives_tag_derivation(self): assert c.is_wheel is True assert c.wheel_tags is not None assert len(c.wheel_tags) >= 1 + + +# --------------------------------------------------------------------------- +# T_M1 — Requires-Python end-to-end preservation (Initiative G Phase 3b) +# --------------------------------------------------------------------------- +# +# T_M3 emits ``markers="python_version >= '3.10'"`` on a ``LockedRequirement`` +# whenever the resolved candidate carries a Requires-Python specifier. +# That round-trip depends on ``Candidate.requires_python`` flowing +# unchanged from the PEP 691 simple-API parse, through the candidate +# cache, into the resolver. T_M1's audit confirmed the field already +# survives end-to-end (smoke: ``click 8.3.3`` came out with +# ``requires_python=">=3.10"``); the test below pins that behaviour so +# a future refactor cannot silently regress the field. + + +class TestRequiresPythonPreservation: + """Pin :attr:`Candidate.requires_python` preservation through the + PEP 691 simple-API parse — T_M3 emits its marker output from this + field.""" + + def test_candidate_requires_python_preserved_from_pep691_parse(self): + """A synthetic PEP 691 JSON page with ``requires-python: ">=3.10"`` + on a single wheel entry yields a :class:`Candidate` whose + :attr:`requires_python` equals the original string verbatim. + + Covers the parse-side of the T_M3 marker-emission dependency: + the in-tree resolver reads ``candidate.requires_python``, so if + the simple-API parser ever drops or normalises the field, the + lockfile-marker output regresses silently. + """ + import json + + from pipenv.resolver.pep691 import _parse_pep691_json + + page = { + "meta": {"api-version": "1.0"}, + "name": "click", + "files": [ + { + "filename": ( + "click-8.3.3-py3-none-any.whl" + ), + "url": ( + "https://files.pythonhosted.org/packages/ab/cd/" + "click-8.3.3-py3-none-any.whl" + ), + "hashes": {"sha256": "0" * 64}, + "requires-python": ">=3.10", + "yanked": False, + } + ], + } + body = json.dumps(page).encode("utf-8") + candidates = _parse_pep691_json( + body, "https://pypi.org/simple/click/" + ) + + assert len(candidates) == 1 + candidate = candidates[0] + assert candidate.name == "click" + assert candidate.version == "8.3.3" + # The exact string is what T_M3 will convert into + # ``python_version >= '3.10'`` on the LockedRequirement entry. + assert candidate.requires_python == ">=3.10" + + def test_candidate_requires_python_none_preserved_from_pep691_parse( + self, + ): + """A PEP 691 page entry without ``requires-python`` produces + :attr:`Candidate.requires_python` ``= None`` — T_M3 omits a + ``python_version`` marker clause in that case.""" + import json + + from pipenv.resolver.pep691 import _parse_pep691_json + + page = { + "meta": {"api-version": "1.0"}, + "name": "six", + "files": [ + { + "filename": "six-1.16.0-py2.py3-none-any.whl", + "url": ( + "https://files.pythonhosted.org/packages/ef/gh/" + "six-1.16.0-py2.py3-none-any.whl" + ), + "hashes": {"sha256": "1" * 64}, + "yanked": False, + } + ], + } + body = json.dumps(page).encode("utf-8") + candidates = _parse_pep691_json( + body, "https://pypi.org/simple/six/" + ) + + assert len(candidates) == 1 + assert candidates[0].requires_python is None diff --git a/tests/unit/test_credential_safety.py b/tests/unit/test_credential_safety.py index 89a7a11da6..4dfb5bd731 100644 --- a/tests/unit/test_credential_safety.py +++ b/tests/unit/test_credential_safety.py @@ -129,6 +129,73 @@ def test_write_credentials_netrc_decodes_url_encoded_password(tmp_path): assert "password p@ss!" in contents +@pytest.mark.utils +def test_write_credentials_netrc_pipfile_wins_over_existing_system_netrc( + tmp_path, monkeypatch +): + """Pipfile-derived creds must beat a duplicate entry from the user's + pre-existing system netrc. + + Regression test for gh-6670: after GHSA-8xgg-v3jj-95m2 moved auth from + URL-embedded argv onto a merged netrc, a stale ``machine`` block in + ``~/.netrc`` (or ``$NETRC``) for the same host silently overrode the + Pipfile's credentials because ``netrc.authenticators()`` returns the + LAST matching entry. Our blocks must come AFTER the appended existing + content so they win that tie-break. + """ + import netrc + + system_netrc = tmp_path / "system.netrc" + system_netrc.write_text( + "machine private.example.com\n login STALE\n password STALE\n", + encoding="utf-8", + ) + monkeypatch.setenv("NETRC", str(system_netrc)) + + sources = [{"url": "https://fresh:freshpass@private.example.com/simple"}] + netrc_path = write_credentials_netrc(sources, tmp_path) + assert netrc_path is not None + + login, _account, password = netrc.netrc(netrc_path).authenticators( + "private.example.com" + ) + assert (login, password) == ("fresh", "freshpass"), ( + "Pipfile-supplied creds must override the user's existing netrc" + ) + + +@pytest.mark.utils +def test_write_credentials_netrc_from_env_var_expanded_url(tmp_path, monkeypatch): + """The expand_url_credentials → write_credentials_netrc round-trip must + produce a netrc with the expanded (real) credentials, not the literal + ``${VAR}`` placeholders. + + Regression test for gh-6670: ensures that env-var-bearing source URLs + flow through ``pipfile_sources()`` (which calls + ``expand_url_credentials``) and yield a netrc with the actual user/pass, + matching the hardcoded-credentials behavior the GHSA fix preserved. + """ + import netrc + + from pipenv.utils.shell import expand_url_credentials + + monkeypatch.setenv("NEXUS_USERNAME", "real-user") + monkeypatch.setenv("NEXUS_PASSWORD", "real-pass!@#") + + raw_url = ( + "https://${NEXUS_USERNAME}:${NEXUS_PASSWORD}" + "@nexus.example.com/repository/pypi/simple" + ) + expanded = expand_url_credentials(raw_url) + netrc_path = write_credentials_netrc([{"url": expanded}], tmp_path) + assert netrc_path is not None + + login, _account, password = netrc.netrc(netrc_path).authenticators( + "nexus.example.com" + ) + assert (login, password) == ("real-user", "real-pass!@#") + + # --- pip_install_deps integration ------------------------------------------- diff --git a/tests/unit/test_pipfile_subsystem.py b/tests/unit/test_pipfile_subsystem.py index 066591bd97..da62ba0610 100644 --- a/tests/unit/test_pipfile_subsystem.py +++ b/tests/unit/test_pipfile_subsystem.py @@ -487,3 +487,78 @@ def test_ensure_proper_casing_returns_false_when_no_change_needed( # when proper_case can't resolve a name). changed = project_with_packages.pipfile.ensure_proper_casing() assert changed is False + + +@pytest.mark.utils +def test_recase_is_noop_by_default(project_with_packages): + # Default mode (``[pipenv] package_name_case`` unset) must not touch + # the network — protects ``pipenv install -r`` from per-package PyPI + # HTTP probes. + with mock.patch("pipenv.utils.pipfile.proper_case") as probe: + project_with_packages.pipfile.recase() + probe.assert_not_called() + + +def _override_settings(project, mapping: dict) -> None: + """Replace the cached ``project.settings`` mapping for one test. + + ``Project.settings`` is a ``@cached_property``; assigning the same + name on the instance ``__dict__`` shadows the descriptor for the + rest of the project's lifetime, which is exactly the scope we want + for a single fixture-bound test. + """ + project.__dict__["settings"] = mapping + + +@pytest.mark.utils +def test_recase_canonical_mode_lowercases_offline(project_with_packages): + pf = project_with_packages.pipfile + pf.parsed.setdefault("packages", {})["Pillow"] = "*" + _override_settings( + project_with_packages, {"package_name_case": "canonical"} + ) + with mock.patch("pipenv.utils.pipfile.proper_case") as probe: + pf.recase() + probe.assert_not_called() + assert "Pillow" not in pf.parsed["packages"] + assert "pillow" in pf.parsed["packages"] + + +@pytest.mark.utils +def test_recase_pypi_mode_invokes_probe(project_with_packages): + pf = project_with_packages.pipfile + _override_settings( + project_with_packages, {"package_name_case": "pypi"} + ) + with mock.patch( + "pipenv.utils.pipfile.proper_case", side_effect=OSError + ) as probe: + pf.recase() + assert probe.called + + +@pytest.mark.utils +@pytest.mark.parametrize( + "raw,expected", + [ + (None, "off"), + ("", "off"), + ("off", "off"), + ("OFF", "off"), + ("none", "off"), + ("no", "off"), + ("false", "off"), + ("0", "off"), + ("canonical", "canonical"), + ("CANONICAL", "canonical"), + ("pypi", "pypi"), + ("PyPI", "pypi"), + ("garbage", "off"), + (False, "off"), + (True, "canonical"), + ], +) +def test_normalize_package_name_case(raw, expected): + from pipenv.utils.pipfile import _normalize_package_name_case + + assert _normalize_package_name_case(raw) == expected diff --git a/tests/unit/test_prefetch.py b/tests/unit/test_prefetch.py new file mode 100644 index 0000000000..6caa9b0ffd --- /dev/null +++ b/tests/unit/test_prefetch.py @@ -0,0 +1,482 @@ +"""Unit tests for :mod:`pipenv.utils.prefetch`. + +Scope (per the perf-pre-fetch landing): + +* Happy path — three deps with matching tags + hashes get downloaded + to the returned dir. +* Empty-deps / no-sources / no-hashes fast paths return ``None`` + without touching the network. +* Hash mismatch on a downloaded wheel → that wheel is dropped (file + not in the result dir) without breaking the rest of the batch. +* Internal helper :func:`_pick_wheel` skips wheels whose tags don't + match the target Python AND whose hashes are absent from the + lockfile entry — both filters are load-bearing. + +The full network path is mocked at the ``urllib3.PoolManager`` and +``ParallelFetcher`` seams so the suite stays offline and fast. +""" +from __future__ import annotations + +import hashlib +from pathlib import Path +from unittest.mock import MagicMock, patch + +from pipenv.utils.prefetch import ( + _download_and_verify, + _pick_wheel, + prefetch_wheels, +) + + +def _sha256_hex(body: bytes) -> str: + return hashlib.sha256(body).hexdigest() + + +def _fake_response(*, status: int, body: bytes): + """Build an object matching urllib3's response shape used by the helper.""" + resp = MagicMock() + resp.status = status + resp.data = body + resp.release_conn = MagicMock() + return resp + + +# --------------------------------------------------------------------------- +# _download_and_verify — direct unit +# --------------------------------------------------------------------------- + + +class TestDownloadAndVerify: + def test_happy_path_writes_file(self, tmp_path): + body = b"fake-wheel-bytes" + expected = f"sha256:{_sha256_hex(body)}" + session = MagicMock() + session.request.return_value = _fake_response(status=200, body=body) + + ok = _download_and_verify( + session, + ("https://pypi/foo.whl", "foo-1.0-py3-none-any.whl", expected), + tmp_path, + ) + assert ok is True + assert (tmp_path / "foo-1.0-py3-none-any.whl").read_bytes() == body + + def test_hash_mismatch_drops_file(self, tmp_path): + body = b"wrong-bytes" + # Expected sha256 of a DIFFERENT body — should not match. + expected = f"sha256:{_sha256_hex(b'right-bytes')}" + session = MagicMock() + session.request.return_value = _fake_response(status=200, body=body) + + ok = _download_and_verify( + session, + ("https://pypi/foo.whl", "foo-1.0-py3-none-any.whl", expected), + tmp_path, + ) + assert ok is False + assert not (tmp_path / "foo-1.0-py3-none-any.whl").exists() + + def test_non_200_drops_file(self, tmp_path): + session = MagicMock() + session.request.return_value = _fake_response(status=404, body=b"") + ok = _download_and_verify( + session, + ("https://pypi/foo.whl", "foo-1.0-py3-none-any.whl", "sha256:x"), + tmp_path, + ) + assert ok is False + + def test_session_raises_returns_false(self, tmp_path): + session = MagicMock() + session.request.side_effect = RuntimeError("synthetic network") + ok = _download_and_verify( + session, + ("https://pypi/foo.whl", "foo-1.0-py3-none-any.whl", "sha256:x"), + tmp_path, + ) + assert ok is False + + def test_path_traversal_filename_rejected(self, tmp_path): + body = b"x" + expected = f"sha256:{_sha256_hex(body)}" + session = MagicMock() + session.request.return_value = _fake_response(status=200, body=body) + # Filename with path separator must be coerced to its basename + # OR rejected — either way no escape outside ``tmp_path``. + ok = _download_and_verify( + session, + ("https://pypi/x", "../escape.whl", expected), + tmp_path, + ) + # Either rejected outright (returns False) OR written to + # ``tmp_path/escape.whl`` (basename-only). In neither case + # does an ``../escape.whl`` appear outside ``tmp_path``. + outside = tmp_path.parent / "escape.whl" + assert not outside.exists() + if ok: + assert (tmp_path / "escape.whl").exists() + + +# --------------------------------------------------------------------------- +# _pick_wheel — tag + hash filtering +# --------------------------------------------------------------------------- + + +def _hash(value: str): + h = MagicMock() + h.algo = "sha256" + h.value = value + return h + + +def _tag(interp: str, abi: str, platform: str): + """Build a duck-typed Tag-like object exposing the str attrs we read.""" + t = MagicMock() + t.interpreter = interp + t.abi = abi + t.platform = platform + return t + + +def _candidate( + *, + version: str, + is_wheel: bool = True, + yanked: bool = False, + wheel_tags=None, + hashes=(), + url: str = "https://pypi/foo.whl", + filename: str = "foo-1.0-cp311-cp311-linux_x86_64.whl", +): + c = MagicMock() + c.version = version + c.is_wheel = is_wheel + c.yanked = yanked + c.wheel_tags = frozenset(wheel_tags or ()) + c.hashes = tuple(hashes) + c.url = url + c.filename = filename + return c + + +def _fake_cache(manifest_by_name: dict): + """Build a fake :class:`ParsedManifestCache` reading from a dict.""" + class _C: + def get(self, _index_url, name): + m = manifest_by_name.get(name) + if m is None: + return None + wrapper = MagicMock() + wrapper.candidates = m + return wrapper + + return _C() + + +class TestPickWheel: + def test_returns_url_filename_hash_on_match(self): + target_tags = frozenset({("cp311", "cp311", "linux_x86_64")}) + cand = _candidate( + version="1.0", + wheel_tags=[_tag("cp311", "cp311", "linux_x86_64")], + hashes=[_hash("abc123")], + ) + cache = _fake_cache({"foo": [cand]}) + result = _pick_wheel( + "foo", "1.0", {"sha256:abc123"}, cache, ["https://idx"], target_tags + ) + assert result == ( + "https://pypi/foo.whl", + "foo-1.0-cp311-cp311-linux_x86_64.whl", + "sha256:abc123", + ) + + def test_skips_wrong_version(self): + target_tags = frozenset({("cp311", "cp311", "linux_x86_64")}) + cand = _candidate( + version="2.0", + wheel_tags=[_tag("cp311", "cp311", "linux_x86_64")], + hashes=[_hash("abc")], + ) + cache = _fake_cache({"foo": [cand]}) + assert _pick_wheel( + "foo", "1.0", {"sha256:abc"}, cache, ["https://idx"], target_tags + ) is None + + def test_skips_sdists(self): + target_tags = frozenset({("cp311", "cp311", "linux_x86_64")}) + cand = _candidate( + version="1.0", is_wheel=False, hashes=[_hash("abc")] + ) + cache = _fake_cache({"foo": [cand]}) + assert _pick_wheel( + "foo", "1.0", {"sha256:abc"}, cache, ["https://idx"], target_tags + ) is None + + def test_skips_yanked(self): + target_tags = frozenset({("cp311", "cp311", "linux_x86_64")}) + cand = _candidate( + version="1.0", + yanked=True, + wheel_tags=[_tag("cp311", "cp311", "linux_x86_64")], + hashes=[_hash("abc")], + ) + cache = _fake_cache({"foo": [cand]}) + assert _pick_wheel( + "foo", "1.0", {"sha256:abc"}, cache, ["https://idx"], target_tags + ) is None + + def test_skips_wrong_platform(self): + # Wheel tags say cp310-macosx; target says cp311-linux. + target_tags = frozenset({("cp311", "cp311", "linux_x86_64")}) + cand = _candidate( + version="1.0", + wheel_tags=[_tag("cp310", "cp310", "macosx_10_9_x86_64")], + hashes=[_hash("abc")], + ) + cache = _fake_cache({"foo": [cand]}) + assert _pick_wheel( + "foo", "1.0", {"sha256:abc"}, cache, ["https://idx"], target_tags + ) is None + + def test_skips_when_hash_not_in_lockfile(self): + # The lockfile pins a DIFFERENT hash from what the candidate + # advertises — could be a stale cache or a poisoned index. + # Drop the wheel; pip handles via the regular index path. + target_tags = frozenset({("cp311", "cp311", "linux_x86_64")}) + cand = _candidate( + version="1.0", + wheel_tags=[_tag("cp311", "cp311", "linux_x86_64")], + hashes=[_hash("other-hash")], + ) + cache = _fake_cache({"foo": [cand]}) + assert _pick_wheel( + "foo", "1.0", {"sha256:abc"}, cache, ["https://idx"], target_tags + ) is None + + +# --------------------------------------------------------------------------- +# prefetch_wheels — top-level integration with mocks +# --------------------------------------------------------------------------- + + +class TestPrefetchWheelsFastPaths: + """Empty / missing inputs return ``None`` without any network.""" + + def test_empty_deps_returns_none(self): + result = prefetch_wheels( + project=MagicMock(), + deps=[], + lockfile_section={}, + sources=[{"name": "pypi", "url": "https://pypi.org/simple"}], + ) + assert result is None + + def test_no_sources_returns_none(self): + dep = MagicMock(name="dep") + dep.name = "foo" + result = prefetch_wheels( + project=MagicMock(), + deps=[(dep, "foo==1.0")], + lockfile_section={ + "foo": {"version": "==1.0", "hashes": ["sha256:abc"]} + }, + sources=[], + ) + assert result is None + + def test_entries_without_hashes_skipped(self): + # A lockfile entry without ``hashes`` (legacy or hand-written) + # is unverifiable; pre-fetch falls through. With zero + # verifiable targets the whole pre-fetch returns ``None``. + dep = MagicMock() + dep.name = "foo" + result = prefetch_wheels( + project=MagicMock(), + deps=[(dep, "foo==1.0")], + lockfile_section={"foo": {"version": "==1.0", "hashes": []}}, + sources=[{"name": "pypi", "url": "https://pypi.org/simple"}], + ) + assert result is None + + +class TestPrefetchWheelsHappyPath: + """Mock the network seams and verify the orchestration writes the + expected wheel files to a returned dir.""" + + def test_two_packages_downloaded_in_parallel(self, tmp_path, monkeypatch): + # Build matching candidate hashes. + foo_body = b"foo-wheel-body" + bar_body = b"bar-wheel-body" + foo_hash = f"sha256:{_sha256_hex(foo_body)}" + bar_hash = f"sha256:{_sha256_hex(bar_body)}" + + # Fake target tags — both candidates have one matching tag. + target_tags = frozenset({("cp311", "cp311", "linux_x86_64")}) + + foo_cand = _candidate( + version="1.0", + wheel_tags=[_tag("cp311", "cp311", "linux_x86_64")], + hashes=[_hash(foo_hash.split(":", 1)[1])], + url="https://pypi/foo.whl", + filename="foo-1.0-cp311-cp311-linux_x86_64.whl", + ) + bar_cand = _candidate( + version="2.0", + wheel_tags=[_tag("cp311", "cp311", "linux_x86_64")], + hashes=[_hash(bar_hash.split(":", 1)[1])], + url="https://pypi/bar.whl", + filename="bar-2.0-cp311-cp311-linux_x86_64.whl", + ) + fake_cache = _fake_cache({"foo": [foo_cand], "bar": [bar_cand]}) + + # Patch the three external seams: PoolManager, PEP691Client, + # ParsedManifestCache, ParallelFetcher, target-tag query. + monkeypatch.setattr( + "pipenv.utils.prefetch._query_target_tags", + lambda *_args, **_kw: target_tags, + ) + + # Build a fake urllib3 PoolManager that returns body by URL. + body_by_url = { + "https://pypi/foo.whl": foo_body, + "https://pypi/bar.whl": bar_body, + } + + class _FakeSession: + def request(self, method, url, **_kw): + return _fake_response( + status=200, body=body_by_url.get(url, b"") + ) + + def clear(self): + pass + + monkeypatch.setattr( + "pipenv.patched.pip._vendor.urllib3.PoolManager", + lambda **_kw: _FakeSession(), + ) + monkeypatch.setattr( + "pipenv.resolver.pep691.PEP691Client", + lambda *a, **k: MagicMock(), + ) + monkeypatch.setattr( + "pipenv.resolver.manifest_cache.ParsedManifestCache", + lambda *a, **k: fake_cache, + ) + + class _FakeFetcher: + def __init__(self, *a, **k): + pass + + def populate(self, _targets): + return {} + + monkeypatch.setattr( + "pipenv.resolver.fetcher.ParallelFetcher", _FakeFetcher + ) + monkeypatch.setattr( + "pipenv.utils.shell.project_python", + lambda *a, **k: "/usr/bin/python3", + ) + + dep_foo = MagicMock() + dep_foo.name = "foo" + dep_bar = MagicMock() + dep_bar.name = "bar" + + result = prefetch_wheels( + project=MagicMock(), + deps=[(dep_foo, "foo==1.0"), (dep_bar, "bar==2.0")], + lockfile_section={ + "foo": {"version": "==1.0", "hashes": [foo_hash]}, + "bar": {"version": "==2.0", "hashes": [bar_hash]}, + }, + sources=[{"name": "pypi", "url": "https://pypi.org/simple"}], + ) + + assert result is not None + result_dir = Path(result) + assert (result_dir / "foo-1.0-cp311-cp311-linux_x86_64.whl").read_bytes() == foo_body + assert (result_dir / "bar-2.0-cp311-cp311-linux_x86_64.whl").read_bytes() == bar_body + + def test_no_matching_wheel_returns_none(self, tmp_path, monkeypatch): + # Target tag doesn't match any candidate → pre-fetch returns + # None (nothing to share) and CLEANS UP the temp dir. + target_tags = frozenset({("cp312", "cp312", "linux_x86_64")}) + monkeypatch.setattr( + "pipenv.utils.prefetch._query_target_tags", + lambda *_args, **_kw: target_tags, + ) + + # Candidate is cp310 — won't match cp312 target. + cand = _candidate( + version="1.0", + wheel_tags=[_tag("cp310", "cp310", "linux_x86_64")], + hashes=[_hash("abc")], + ) + monkeypatch.setattr( + "pipenv.patched.pip._vendor.urllib3.PoolManager", + lambda **_kw: MagicMock(), + ) + monkeypatch.setattr( + "pipenv.resolver.pep691.PEP691Client", + lambda *a, **k: MagicMock(), + ) + monkeypatch.setattr( + "pipenv.resolver.manifest_cache.ParsedManifestCache", + lambda *a, **k: _fake_cache({"foo": [cand]}), + ) + + class _FakeFetcher: + def __init__(self, *a, **k): + pass + + def populate(self, _targets): + return {} + + monkeypatch.setattr( + "pipenv.resolver.fetcher.ParallelFetcher", _FakeFetcher + ) + monkeypatch.setattr( + "pipenv.utils.shell.project_python", + lambda *a, **k: "/usr/bin/python3", + ) + + dep = MagicMock() + dep.name = "foo" + result = prefetch_wheels( + project=MagicMock(), + deps=[(dep, "foo==1.0")], + lockfile_section={ + "foo": {"version": "==1.0", "hashes": ["sha256:abc"]} + }, + sources=[{"name": "pypi", "url": "https://pypi.org/simple"}], + ) + assert result is None + + def test_target_tag_query_failure_returns_none(self, monkeypatch): + """When the target-tag subprocess fails (timeout, non-zero + exit), pre-fetch bails rather than risk downloading wheels + for the wrong platform. Defensive — pip handles the install + via the regular index path.""" + monkeypatch.setattr( + "pipenv.utils.prefetch._query_target_tags", + lambda *_args, **_kw: None, + ) + monkeypatch.setattr( + "pipenv.utils.shell.project_python", + lambda *a, **k: "/usr/bin/python3", + ) + + dep = MagicMock() + dep.name = "foo" + result = prefetch_wheels( + project=MagicMock(), + deps=[(dep, "foo==1.0")], + lockfile_section={ + "foo": {"version": "==1.0", "hashes": ["sha256:abc"]} + }, + sources=[{"name": "pypi", "url": "https://pypi.org/simple"}], + ) + assert result is None diff --git a/tests/unit/test_pure_python_backend.py b/tests/unit/test_pure_python_backend.py new file mode 100644 index 0000000000..04969d266a --- /dev/null +++ b/tests/unit/test_pure_python_backend.py @@ -0,0 +1,2524 @@ +"""Unit tests for :class:`PurePythonBackend` (Initiative G phase 3, T9). + +The backend wraps the :class:`PurePythonProvider` (T3-T8) + +:class:`MetadataFetcher` (T2) + :class:`Requirement` (T1) chain in the +Initiative F ``Backend`` protocol, with the following Phase 3-specific +behaviours validated below: + +* **Happy path** — :func:`_drive_resolver` returns a synthetic result; + the backend translates the resolved candidate mapping into a typed + :class:`ResolverSuccess` payload. +* **Resolution conflict** (:class:`ResolutionImpossible`) — translated + into a :class:`ResolutionError` whose ``pip_message`` names BOTH + conflicting causes (so the user can find the bad pin without reading + resolvelib internals). +* **Sdist transitive transparent resolution** (Phase 3b T_S3) — when + resolvelib expands a transitive whose only artifact is an sdist, + the provider routes it through ``MetadataFetcher.fetch_metadata`` + (T_S2) which builds METADATA via PEP 517 (T_S1) and resolution + proceeds normally. This replaced the Phase 3a fail-loud + :class:`_SdistEncountered` path which raised an + :class:`InternalError`. +* **Top-level emptiness pre-check** (Phase 3b T_S4) — when a top-level + package has ZERO candidates across every configured index (typo / + yanked-only / cold-cache + every-fetch-failed), the backend aborts + BEFORE driving resolvelib, returning a :class:`ResolutionError` + whose ``pip_message`` names the offending package and suggests + ``pipenv lock --backend pip``. (Phase 3a fired the same gate on + sdist-only top-levels; T_S2/T_S3 made sdists transparently + resolvable so that branch is gone — see + ``TestQFPreCheck.test_sdist_only_top_level_resolves_after_t_s4``.) + +Mocks everything (no HTTP, no real resolvelib drive). +""" +from __future__ import annotations + +from typing import Any +from unittest import mock + +import pytest + +from pipenv.resolver.candidate import Candidate, Hash +from pipenv.resolver.schema import ( + SCHEMA_VERSION, + InternalError, + PackageSpecs, + ResolutionError, + ResolverOptions, + ResolverRequest, + ResolverResponse, + ResolverSuccess, + Source, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +_INDEX = "https://pypi.org/simple" + + +def _build_request(specs: dict[str, str] | None = None) -> ResolverRequest: + """Build a minimal but realistic :class:`ResolverRequest`.""" + if specs is None: + specs = {"requests": "requests==2.31.0"} + return ResolverRequest( + schema_version=SCHEMA_VERSION, + category="default", + packages=PackageSpecs(specs=specs), + options=ResolverOptions(), + sources=( + Source(name="pypi", url=_INDEX, verify_ssl=True), + ), + ) + + +def _wheel_candidate(name: str, version: str) -> Candidate: + filename = f"{name}-{version}-py3-none-any.whl" + return Candidate( + name=name, + version=version, + url=f"{_INDEX}/{name}/{filename}", + filename=filename, + hashes=frozenset({Hash("sha256", "0" * 64)}), + requires_python=None, + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=True, + wheel_tags=None, + extras=frozenset(), + ) + + +def _sdist_candidate(name: str, version: str) -> Candidate: + filename = f"{name}-{version}.tar.gz" + return Candidate( + name=name, + version=version, + url=f"{_INDEX}/{name}/{filename}", + filename=filename, + hashes=frozenset({Hash("sha256", "1" * 64)}), + requires_python=None, + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=False, + wheel_tags=None, + extras=frozenset(), + ) + + +class _FakeManifest: + __slots__ = ("candidates",) + + def __init__(self, candidates: tuple[Candidate, ...]) -> None: + self.candidates = candidates + + +class _FakeCache: + """In-memory stand-in for :class:`ParsedManifestCache`.""" + + def __init__(self, mapping: dict[tuple[str, str], tuple[Candidate, ...]]) -> None: + self._mapping = dict(mapping) + + def get(self, index_url: str, package_name: str): + cands = self._mapping.get((index_url, package_name)) + if cands is None: + return None + return _FakeManifest(cands) + + +class _FakeFetcher: + """Stand-in for :class:`ParallelFetcher` that records calls.""" + + def __init__(self) -> None: + self.populate_calls: list = [] + + def populate(self, targets): + self.populate_calls.append(list(targets)) + return {} + + +class _FakeResult: + """Synthetic stand-in for ``resolvelib.Result`` carrying ``.mapping`` + and optionally ``.criteria`` (T_M3 — marker emission reads + ``Result.criteria[identifier].information``). + """ + + def __init__(self, mapping: dict, criteria: dict | None = None) -> None: + self.mapping = mapping + self.criteria = criteria or {} + + +class _FakeCriterion: + """Synthetic stand-in for ``resolvelib.resolvers.criterion.Criterion``; + only the ``.information`` collection is read by T_M3's translator. + """ + + def __init__(self, information) -> None: + self.information = tuple(information) + + +# --------------------------------------------------------------------------- +# T9 — Acceptance tests +# --------------------------------------------------------------------------- + + +class TestSuccessPath: + """Mock :func:`_drive_resolver` to return a synthetic result with + resolved candidates → backend returns :class:`ResolverSuccess`. + """ + + def test_success_translates_mapping_into_locked_tuple(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + # Cache has wheel candidates for every top-level so the Q-F + # pre-check passes and resolution proceeds. + cache = _FakeCache( + { + (_INDEX, "requests"): ( + _wheel_candidate("requests", "2.31.0"), + ), + } + ) + fetcher = _FakeFetcher() + + # Synthetic resolved mapping: identifier -> Candidate. + resolved_candidate = _wheel_candidate("requests", "2.31.0") + fake_result = _FakeResult( + mapping={ + ("requests", frozenset()): resolved_candidate, + } + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=fetcher, + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"requests": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ) as drive: + response = backend.resolve(request) + + assert drive.called + assert isinstance(response, ResolverResponse) + assert isinstance(response.result, ResolverSuccess) + locked = list(response.result.locked) + assert len(locked) == 1 + assert locked[0].name == "requests" + # Lockfile shape uses the ``==`` prefix per the + # existing pip-backend formatter (see ``_clean_version`` + # auto-prefix at ``schema.py`` lines 213-224). The pure-python + # backend mirrors that convention so downstream consumers + # (lockfile writer / parity tests) don't see a divergent shape. + assert locked[0].version == "==2.31.0" + + +class TestResolutionImpossible: + """Mock :func:`_drive_resolver` to raise :class:`ResolutionImpossible` + → backend returns :class:`ResolutionError` whose ``pip_message`` + mentions BOTH conflicting causes. + """ + + def test_resolution_impossible_translates_to_resolution_error(self): + from pipenv.patched.pip._vendor.resolvelib import ResolutionImpossible + from pipenv.patched.pip._vendor.resolvelib.structs import ( + RequirementInformation, + ) + + from pipenv.resolver.backends.pure_python import PurePythonBackend + from pipenv.resolver.pure_python_requirement import Requirement + + cache = _FakeCache( + { + (_INDEX, "a"): (_wheel_candidate("a", "1.0.0"),), + (_INDEX, "c"): (_wheel_candidate("c", "1.0.0"),), + } + ) + fetcher = _FakeFetcher() + + # Build two conflicting RequirementInformation rows: a → b<2, + # c → b>=2. Parent objects expose ``.name`` (Candidate + # duck-shape) so the formatter can emit "a 1.0.0 requires b<2". + req_b_lt2 = Requirement.from_pipfile_entry("b", "<2") + req_b_ge2 = Requirement.from_pipfile_entry("b", ">=2") + parent_a = mock.MagicMock(name="a", version="1.0.0") + parent_a.name = "a" + parent_a.version = "1.0.0" + parent_c = mock.MagicMock(name="c", version="1.0.0") + parent_c.name = "c" + parent_c.version = "1.0.0" + + causes = [ + RequirementInformation(requirement=req_b_lt2, parent=parent_a), + RequirementInformation(requirement=req_b_ge2, parent=parent_c), + ] + impossible = ResolutionImpossible(causes) + + backend = PurePythonBackend( + cache=cache, + fetcher=fetcher, + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"a": "*", "c": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + side_effect=impossible, + ): + response = backend.resolve(request) + + assert isinstance(response, ResolverResponse) + assert isinstance(response.result, ResolutionError) + msg = response.result.pip_message + # Both conflicting causes must appear in the formatted message + # so the user can find the offending pins without reading + # resolvelib internals. + assert "a" in msg + assert "c" in msg + assert "<2" in msg + assert ">=2" in msg + + +class TestSdistTransitiveResolvesThroughMetadataFetcher: + """Phase 3b T_S3: a transitive sdist no longer crashes the backend. + + The provider's :meth:`get_dependencies` now routes every candidate + through ``self._metadata_fetcher`` regardless of ``is_wheel`` — + T_S2's :meth:`MetadataFetcher.fetch_metadata` branches on + ``is_wheel`` and builds sdist METADATA via T_S1's PEP 517 path. + + The Phase 3a ``_SdistEncountered`` → :class:`InternalError` branch + in :meth:`PurePythonBackend.resolve` is gone; this test pins the + new behaviour by feeding the backend a synthetic resolvelib + ``Result`` whose ``mapping`` includes a sdist candidate alongside + a wheel. The translator must produce :class:`ResolverSuccess` + with both entries locked. + + Critically: a Phase 3a regression here would have produced an + :class:`InternalError`; we assert :class:`ResolverSuccess`. + """ + + def test_sdist_transitive_resolves_through_metadata_fetcher(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + cache = _FakeCache( + { + # Top-level "broken" has a wheel candidate so the Q-F + # pre-check passes; the sdist appears as a TRANSITIVE. + (_INDEX, "broken"): (_wheel_candidate("broken", "1.0.0"),), + } + ) + fetcher = _FakeFetcher() + + # Synthetic resolved mapping: the top-level wheel plus a + # transitive sdist that — Phase 3a — would have raised + # _SdistEncountered. T_S3 means the provider expanded it + # transparently via T_S2 and resolvelib produced a clean + # mapping. + resolved_top = _wheel_candidate("broken", "1.0.0") + resolved_sdist = _sdist_candidate("legacy-dep", "0.9.0") + fake_result = _FakeResult( + mapping={ + ("broken", frozenset()): resolved_top, + ("legacy-dep", frozenset()): resolved_sdist, + } + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=fetcher, + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"broken": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ) as drive: + response = backend.resolve(request) + + assert drive.called + assert isinstance(response, ResolverResponse) + # Post-T_S3: ResolverSuccess, NOT InternalError. The transitive + # sdist did not derail resolution. + assert isinstance(response.result, ResolverSuccess) + locked_names = sorted(lr.name for lr in response.result.locked) + assert locked_names == ["broken", "legacy-dep"] + # The sdist transitive is locked at its version per the + # standard ``==`` translation; pin so a regression + # that silently drops the sdist entry is caught. + sdist_entry = next( + lr for lr in response.result.locked if lr.name == "legacy-dep" + ) + assert sdist_entry.version == "==0.9.0" + + +class TestQFPreCheck: + """Top-level emptiness pre-check (Phase 3b T_S4). + + Two rows pinned here: + + * **Sdist-only top-level** (post-T_S4) — resolution must succeed. + Phase 3a fired ``ResolutionError`` on this shape because no wheel + meant no resolvable METADATA; T_S2 made ``MetadataFetcher`` build + sdist METADATA via PEP 517, so the pre-check no longer treats + sdist-only as a failure. We pin :class:`ResolverSuccess`. + * **Zero candidates across all indexes** — the new pre-check fires + with a ``ResolutionError`` whose message names the offending + package and suggests ``pipenv lock --backend pip``. + :func:`_drive_resolver` is NEVER invoked. + """ + + def test_sdist_only_top_level_resolves_after_t_s4(self): + """Post-T_S4 (Phase 3b): a top-level package with ONLY sdist + candidates must resolve normally — the obsolete Phase 3a + wheel-availability gate is gone. + + The sdist gets built transparently via T_S2's + :class:`MetadataFetcher` route (the actual PEP 517 invocation + is exercised in ``test_pure_python_sdist.py``; here we just + confirm the backend no longer fires the pre-check). + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + # ``brokenpkg`` has only an sdist candidate. In Phase 3a this + # would have fired the Q-F gate; post-T_S4 it must NOT. + cache = _FakeCache( + { + (_INDEX, "brokenpkg"): ( + _sdist_candidate("brokenpkg", "1.0.0"), + ), + } + ) + fetcher = _FakeFetcher() + + # Mock _drive_resolver to return a synthetic success — we are + # pinning the *backend's* behaviour around the pre-check, not + # the full provider chain (which is covered in T_S2/T_S3 + # acceptance tests). The mock proves the pre-check did NOT + # short-circuit and the backend reached the resolution step. + resolved = _sdist_candidate("brokenpkg", "1.0.0") + fake_result = _FakeResult( + mapping={("brokenpkg", frozenset()): resolved} + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=fetcher, + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"brokenpkg": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ) as drive: + response = backend.resolve(request) + + # _drive_resolver WAS called — the pre-check did not + # short-circuit on sdist-only top-level. + assert drive.called + + assert isinstance(response, ResolverResponse) + assert isinstance(response.result, ResolverSuccess) + locked = list(response.result.locked) + assert len(locked) == 1 + assert locked[0].name == "brokenpkg" + + def test_zero_candidates_top_level_aborts_before_drive(self): + """A top-level package with ZERO candidates across every + configured index → backend fires the new emptiness pre-check + and aborts BEFORE driving resolvelib. + + This is the typo / yanked-only / total-network-failure path: + we want a clear "no candidates found" message rather than a + 30-second wait followed by resolvelib's opaque + "no candidates available" error. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + # Empty cache: every (index, name) lookup returns None. + cache = _FakeCache({}) + fetcher = _FakeFetcher() + + backend = PurePythonBackend( + cache=cache, + fetcher=fetcher, + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"brokenpkg": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver" + ) as drive: + response = backend.resolve(request) + + # _drive_resolver MUST NOT have been called — the emptiness + # pre-check short-circuits before resolvelib runs. + assert not drive.called + + assert isinstance(response, ResolverResponse) + assert isinstance(response.result, ResolutionError) + msg = response.result.pip_message + assert "brokenpkg" in msg + assert "no candidates found" in msg + assert "pipenv lock --backend pip" in msg + + +# --------------------------------------------------------------------------- +# T14 — Coverage-completing tests +# --------------------------------------------------------------------------- + + +class TestIsAvailable: + """The pure-python backend has no external dependency to probe for + — it ships in-tree as part of ``pipenv.resolver``. :meth:`is_available` + is documented to always return ``True``; pin that behaviour so a + future refactor can't quietly flip the contract. + """ + + def test_is_available_returns_true(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + backend = PurePythonBackend( + cache=_FakeCache({}), + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + assert backend.is_available() is True + + +class TestPrefetchExceptionTolerated: + """Q-B pre-fetch failures are non-fatal: the provider's lazy + :meth:`find_matches` will retry on cache miss, so the backend + swallows the exception and continues. This pins the + ``except Exception: pass`` branch — without it a transient HTTP + error during pre-fetch would abort resolution that could otherwise + have succeeded against a populated cache. + """ + + def test_populate_exception_does_not_abort_resolve(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + class _ExplodingFetcher: + def __init__(self) -> None: + self.calls = 0 + + def populate(self, targets): + self.calls += 1 + raise RuntimeError("simulated transient network failure") + + cache = _FakeCache( + { + (_INDEX, "requests"): ( + _wheel_candidate("requests", "2.31.0"), + ), + } + ) + fetcher = _ExplodingFetcher() + resolved = _wheel_candidate("requests", "2.31.0") + fake_result = _FakeResult( + mapping={("requests", frozenset()): resolved} + ) + backend = PurePythonBackend( + cache=cache, + fetcher=fetcher, + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"requests": "*"}) + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + # Pre-fetch did blow up exactly once. + assert fetcher.calls == 1 + # …yet the backend still produced a success — pre-fetch failure + # is non-fatal. + assert isinstance(response.result, ResolverSuccess) + + +class TestQFPreCheckEdges: + """Top-level emptiness pre-check edge cases (Phase 3b T_S4) beyond + the two-row baseline covered by ``TestQFPreCheck``. + + Four rows pinned here: + + 1. Mixed sdist + wheel candidates — pre-check must NOT fire (one + top-level has candidates, period; their artifact mix is the + provider's problem now). + 2. Sdist-only candidates across all configured indexes — pre-check + must NOT fire (post-T_S4: sdists are resolvable). + 3. Multi-top-level with two healthy packages + one empty package — + the new pre-check fires and the message names ONLY the empty + one (not the healthy siblings). + 4. Multi-index plural-spelling — "indexes" appears in the error + message when more than one index is configured. + """ + + def test_mixed_sdist_and_wheel_does_not_fire_emptiness(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + # Mixed candidates: a sdist AND a wheel for the same package. + # The emptiness pre-check sees ``saw_any=True`` on the first + # candidate (the sdist) — it never reaches the wheel check. + cache = _FakeCache( + { + (_INDEX, "mixed"): ( + _sdist_candidate("mixed", "1.0.0"), + _wheel_candidate("mixed", "1.0.0"), + ), + } + ) + fetcher = _FakeFetcher() + resolved = _wheel_candidate("mixed", "1.0.0") + fake_result = _FakeResult( + mapping={("mixed", frozenset()): resolved} + ) + backend = PurePythonBackend( + cache=cache, + fetcher=fetcher, + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"mixed": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ) as drive: + response = backend.resolve(request) + + assert drive.called + assert isinstance(response.result, ResolverSuccess) + + def test_sdist_only_across_all_indexes_does_not_fire_emptiness(self): + """Two indexes, both serving only sdists — the emptiness gate + treats those candidates as present and does NOT fire. T_S4 + decoupled the gate from artifact type, so the multi-index + sdist-only shape (Phase 3a's worst-case) now resolves. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + idx_primary = "https://primary.example/simple" + idx_secondary = "https://secondary.example/simple" + + cache = _FakeCache( + { + (idx_primary, "sdistonly"): ( + _sdist_candidate("sdistonly", "1.0.0"), + ), + (idx_secondary, "sdistonly"): ( + _sdist_candidate("sdistonly", "1.0.0"), + ), + } + ) + fetcher = _FakeFetcher() + resolved = _sdist_candidate("sdistonly", "1.0.0") + fake_result = _FakeResult( + mapping={("sdistonly", frozenset()): resolved} + ) + backend = PurePythonBackend( + cache=cache, + fetcher=fetcher, + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = ResolverRequest( + schema_version=SCHEMA_VERSION, + category="default", + packages=PackageSpecs(specs={"sdistonly": "*"}), + options=ResolverOptions(), + sources=( + Source(name="primary", url=idx_primary, verify_ssl=True), + Source(name="secondary", url=idx_secondary, verify_ssl=True), + ), + ) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ) as drive: + response = backend.resolve(request) + + assert drive.called + assert isinstance(response.result, ResolverSuccess) + + def test_mixed_top_level_names_only_empty_one_in_message(self): + """Three top-level packages: two have candidates (wheels), one + has zero across every index. The pre-check must fire naming + ONLY the empty offender — the healthy siblings must not appear + in the error message. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + cache = _FakeCache( + { + (_INDEX, "healthy_one"): ( + _wheel_candidate("healthy_one", "1.0.0"), + ), + (_INDEX, "healthy_two"): ( + _wheel_candidate("healthy_two", "2.0.0"), + ), + # ``ghostpkg`` is missing from the cache entirely. + } + ) + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request( + { + "healthy_one": "*", + "healthy_two": "*", + "ghostpkg": "*", + } + ) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver" + ) as drive: + response = backend.resolve(request) + + # Pre-check short-circuited — resolvelib never ran. + assert not drive.called + assert isinstance(response.result, ResolutionError) + msg = response.result.pip_message + # Only the empty offender is named. + assert "ghostpkg" in msg + assert "healthy_one" not in msg + assert "healthy_two" not in msg + # Single-index spelling (one source on the request). + assert "configured index." in msg or "configured index " in msg + assert "configured indexes" not in msg + + def test_multi_index_uses_plural_spelling(self): + """When more than one index is configured the error message + uses the plural "indexes" form. Pin this small grammatical + detail so a refactor that drops the conditional doesn't leak + ungrammatical English to users. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + idx_a = "https://a.example/simple" + idx_b = "https://b.example/simple" + + cache = _FakeCache({}) # empty across both indexes + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = ResolverRequest( + schema_version=SCHEMA_VERSION, + category="default", + packages=PackageSpecs(specs={"ghostpkg": "*"}), + options=ResolverOptions(), + sources=( + Source(name="a", url=idx_a, verify_ssl=True), + Source(name="b", url=idx_b, verify_ssl=True), + ), + ) + + response = backend.resolve(request) + + assert isinstance(response.result, ResolutionError) + assert "configured indexes" in response.result.pip_message + + +class TestGenericExceptionTranslatedToInternalError: + """A truly unexpected exception out of :func:`_drive_resolver` (not + :class:`ResolutionImpossible`) must be caught and translated into + an :class:`InternalError` with the original message AND a + non-empty traceback. This is the catch-all branch — it stops a + stray bug deep in the provider from crashing the resolver + subprocess with an untranslated stack trace. (Phase 3b T_S3 + removed the Phase 3a :class:`_SdistEncountered` clause that used + to sit between :class:`ResolutionImpossible` and this catch-all.) + """ + + def test_unexpected_exception_yields_internal_error(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + cache = _FakeCache( + { + (_INDEX, "requests"): ( + _wheel_candidate("requests", "2.31.0"), + ), + } + ) + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"requests": "*"}) + + boom = RuntimeError("unexpected provider bug") + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + side_effect=boom, + ): + response = backend.resolve(request) + + assert isinstance(response.result, InternalError) + assert "unexpected provider bug" in response.result.message + # Traceback must be populated so the subprocess parent can log + # the failure for the user. + assert response.result.traceback is not None + assert "RuntimeError" in response.result.traceback + + +class TestResolutionImpossibleFormattingEdges: + """Edge cases of the :class:`ResolutionImpossible` translator. + + * Empty causes list (defensive) — produces a header-only + ``pip_message`` and empty ``conflicts``. + * ``parent is None`` (root requirement) — the formatter must + render ```` rather than crashing on the missing attribute + (line 401). + * Parent object that lacks ``.name`` / ``.version`` — the + formatter falls back to ``str(parent)`` (line 408). This is the + defensive branch for non-Candidate parent shapes the resolvelib + protocol allows. + """ + + def test_empty_causes_yields_header_only_message(self): + from pipenv.patched.pip._vendor.resolvelib import ResolutionImpossible + + from pipenv.resolver.backends.pure_python import PurePythonBackend + + cache = _FakeCache( + {(_INDEX, "x"): (_wheel_candidate("x", "1.0.0"),)} + ) + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"x": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + side_effect=ResolutionImpossible([]), + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolutionError) + # Header still present, no per-cause lines. + assert "Resolution impossible" in response.result.pip_message + assert tuple(response.result.conflicts) == () + + def test_root_parent_renders_as_root(self): + from pipenv.patched.pip._vendor.resolvelib import ResolutionImpossible + from pipenv.patched.pip._vendor.resolvelib.structs import ( + RequirementInformation, + ) + + from pipenv.resolver.backends.pure_python import PurePythonBackend + from pipenv.resolver.pure_python_requirement import Requirement + + cache = _FakeCache( + {(_INDEX, "rootpkg"): (_wheel_candidate("rootpkg", "1.0.0"),)} + ) + req_root = Requirement.from_pipfile_entry("rootpkg", ">=99") + causes = [RequirementInformation(requirement=req_root, parent=None)] + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"rootpkg": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + side_effect=ResolutionImpossible(causes), + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolutionError) + # The root parent renders as the ```` sentinel. + assert "" in response.result.pip_message + assert "rootpkg" in response.result.pip_message + + def test_parent_without_name_or_version_uses_str_fallback(self): + from pipenv.patched.pip._vendor.resolvelib import ResolutionImpossible + from pipenv.patched.pip._vendor.resolvelib.structs import ( + RequirementInformation, + ) + + from pipenv.resolver.backends.pure_python import PurePythonBackend + from pipenv.resolver.pure_python_requirement import Requirement + + cache = _FakeCache( + {(_INDEX, "leafpkg"): (_wheel_candidate("leafpkg", "1.0.0"),)} + ) + + # ``object()`` has no .name/.version — but its repr is stable + # enough that we can find it back in the rendered message. + class _OpaqueParent: + def __str__(self) -> str: # noqa: D401 + return "OPAQUE-PARENT-SENTINEL" + + opaque = _OpaqueParent() + req = Requirement.from_pipfile_entry("leafpkg", ">=1") + causes = [RequirementInformation(requirement=req, parent=opaque)] + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"leafpkg": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + side_effect=ResolutionImpossible(causes), + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolutionError) + # str(parent) fallback wins. + assert "OPAQUE-PARENT-SENTINEL" in response.result.pip_message + + +class TestSpecValueTranslation: + """:meth:`PurePythonBackend._spec_value_to_pipfile_entry` translates + the wire-shape ``pip-install`` argument string into a value the + :class:`Requirement` parser accepts. Three branches are pinned: + + * Empty / whitespace-only value → ``"*"`` (line 329). + * Specifier-bearing value → everything from the first marker char + onward (line 339). + * Bare-name value → ``"*"`` (line 344). + """ + + def test_empty_string_returns_star(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", "" + ) == "*" + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", " " + ) == "*" + # ``None`` is tolerated and treated as empty (defensive). + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", None # type: ignore[arg-type] + ) == "*" + + def test_specifier_bearing_line_strips_to_marker_char(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + # ``==``, ``>=``, ``<=``, ``~=``, ``!=`` are the canonical + # multi-char markers; ``>`` / ``<`` are the single-char + # fallbacks. Each should peel back to the marker. + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", "requests==2.31.0" + ) == "==2.31.0" + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", "requests>=2.0" + ) == ">=2.0" + assert PurePythonBackend._spec_value_to_pipfile_entry( + "django", "django<5" + ) == "<5" + assert PurePythonBackend._spec_value_to_pipfile_entry( + "django", "django>4" + ) == ">4" + assert PurePythonBackend._spec_value_to_pipfile_entry( + "flask", "flask!=2.0" + ) == "!=2.0" + assert PurePythonBackend._spec_value_to_pipfile_entry( + "numpy", "numpy~=1.24" + ) == "~=1.24" + + def test_bare_name_returns_star(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", "requests" + ) == "*" + # Casing is ignored (line 343-344). + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", "Requests" + ) == "*" + + def test_unparseable_falls_back_to_star(self): + """The final fallback (line 345 / the ``return "*"`` at the + very bottom) covers shapes like a stray URL or VCS ref that the + wire doesn't pre-canonicalise. We forward as ``"*"`` and let + the upstream parser pin the dep on its own terms. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + # No marker char, doesn't match the package name → final + # ``return "*"`` fallback. + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", "something-weird" + ) == "*" + + def test_pip_flags_after_specifier_are_stripped(self): + """``convert_deps_to_pip`` emits the full pip-install argv as + the wire-shape ``spec_value`` — including ``-i `` and + any ``--trusted-host``/``--extra-index-url`` flags. These belong + on ``request.options.indexes``, not in the version specifier; + feeding them to ``SpecifierSet`` raises ``InvalidSpecifier``. + + Regression test for the bench-fixture lock failure where + ``urllib3 = {extras = ["brotli"], version = ">=2"}`` produced + the wire-shape ``"urllib3[brotli]>=2 -i https://pypi.org/simple"`` + and the parser tried to treat ``">=2 -i https://pypi.org/simple"`` + as a single version specifier. Extras now surface in the + dict-form return (T_PARITY_REAL fix); CLI flags after the + specifier-bearing token still get dropped. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + assert PurePythonBackend._spec_value_to_pipfile_entry( + "urllib3", "urllib3[brotli]>=2 -i https://pypi.org/simple" + ) == {"version": ">=2", "extras": ["brotli"]} + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", + "requests==2.31.0 --index-url https://internal.example/simple", + ) == "==2.31.0" + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", + "requests --trusted-host pypi.example", + ) == "*" + + def test_extras_with_no_specifier_returns_dict(self): + """``name[extras]`` with no version pin now surfaces the extras + in the dict-form return. Previously the function returned + ``"*"`` (extras silently dropped); the T_PARITY_REAL fix routes + the extras into the Requirement so the resolvelib identifier + becomes ``(name, frozenset({"extra"}))`` instead of + ``(name, frozenset())``. Without this, marker-gated + ``Requires-Dist`` entries (e.g. ``psycopg-binary; extra=binary`` + on ``psycopg[binary]``) silently disappear from the lockfile. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + assert PurePythonBackend._spec_value_to_pipfile_entry( + "urllib3", "urllib3[brotli]" + ) == {"version": "*", "extras": ["brotli"]} + + def test_extras_multi_value_split(self): + """Multiple extras (``pkg[a,b,c]``) round-trip as a list.""" + from pipenv.resolver.backends.pure_python import PurePythonBackend + + assert PurePythonBackend._spec_value_to_pipfile_entry( + "requests", "requests[security,socks]>=2.31" + ) == {"version": ">=2.31", "extras": ["security", "socks"]} + + +class TestTargetEnvCaching: + """:meth:`PurePythonBackend._resolved_target_env` is supposed to + return the explicitly-supplied ``target_env`` verbatim AND lazily + cache the running-interpreter default on first call when no + override was passed. Both branches matter — the explicit override + path is exercised by test fixtures, the default-population path + by production. + """ + + def test_explicit_target_env_returned_verbatim(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + sentinel_env = {"python_version": "3.42", "sys_platform": "atari"} + backend = PurePythonBackend( + cache=_FakeCache({}), + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + target_env=sentinel_env, + ) + assert backend._resolved_target_env() is sentinel_env + # Second call returns the same object — no recompute. + assert backend._resolved_target_env() is sentinel_env + + def test_default_target_env_is_populated_lazily(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + backend = PurePythonBackend( + cache=_FakeCache({}), + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + env1 = backend._resolved_target_env() + # The running interpreter's marker env always defines + # ``python_version`` (PEP 508). We don't need to assert the + # value — only that the default-environment shape is present. + assert "python_version" in env1 + # Cached: the second call returns the same dict instance. + env2 = backend._resolved_target_env() + assert env2 is env1 + + +class TestMetadataFetcherClosure: + """The provider receives a one-argument metadata-fetcher closure + that re-binds the session + metadata_cache from the backend + instance. Pin its forwarding behaviour so a future refactor that + swaps the closure for a method can't silently change semantics — + the existing T2 contract is "given a Candidate, return parsed + metadata using the bound session and read-through cache". + """ + + def test_metadata_fetcher_forwards_to_fetch_metadata(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + cache = _FakeCache( + { + (_INDEX, "requests"): ( + _wheel_candidate("requests", "2.31.0"), + ), + } + ) + session_sentinel = mock.MagicMock(name="session") + metadata_cache_sentinel = mock.MagicMock(name="metadata_cache") + resolved = _wheel_candidate("requests", "2.31.0") + fake_result = _FakeResult( + mapping={("requests", frozenset()): resolved} + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=session_sentinel, + metadata_cache=metadata_cache_sentinel, + ) + request = _build_request({"requests": "*"}) + + captured: dict = {} + + def _drive_capturing(reqs, provider): + # The provider's metadata_fetcher is the closure we want to + # inspect. Invoke it and capture the underlying call. + captured["provider"] = provider + return fake_result + + # Patch ``fetch_metadata`` at its import site inside the + # backend module so the closure dispatches through our spy. + sentinel_metadata = mock.MagicMock(name="CoreMetadata") + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + side_effect=_drive_capturing, + ), mock.patch( + "pipenv.resolver.backends.pure_python.fetch_metadata", + return_value=sentinel_metadata, + ) as fetch_spy: + response = backend.resolve(request) + # Exercise the closure with a synthetic candidate. + probe = _wheel_candidate("probe", "0.1.0") + result = captured["provider"]._metadata_fetcher(probe) + + assert isinstance(response.result, ResolverSuccess) + # The closure forwarded the bound session + metadata_cache. + fetch_spy.assert_called_once_with( + probe, session_sentinel, cache=metadata_cache_sentinel + ) + assert result is sentinel_metadata + + +class TestTranslateMappingEdges: + """:meth:`PurePythonBackend._translate_mapping` edge cases beyond + the happy path in ``TestSuccessPath``. + + * Non-tuple identifier — the safety net at line 457-459 falls back + to ``candidate.name`` + empty extras. + * Candidate with no ``version`` — silently skipped (line 467). + * Candidate with extras + multiple hashes — both populate the + :class:`LockedRequirement`. + """ + + def test_non_tuple_identifier_falls_back_to_candidate_name(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + cache = _FakeCache( + {(_INDEX, "weird"): (_wheel_candidate("weird", "1.0.0"),)} + ) + # Identifier is a BARE STRING, not a (name, frozenset(extras)) + # tuple. resolvelib never produces this shape, but the + # defensive unpacking branch at line 457-459 handles it + # gracefully so test fixtures don't have to bother. + resolved = _wheel_candidate("weird", "1.0.0") + fake_result = _FakeResult(mapping={"weird-as-string": resolved}) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"weird": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + # The fallback picked up the candidate's own name. + assert locked[0].name == "weird" + assert locked[0].extras == () + + def test_candidate_without_version_is_skipped(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + cache = _FakeCache( + {(_INDEX, "broken"): (_wheel_candidate("broken", "1.0.0"),)} + ) + + # A candidate-shaped object whose ``version`` is None — would + # crash the LockedRequirement constructor (which rejects + # version=None + no vcs/file/path). The backend defensively + # skips it. + class _VersionlessCandidate: + name = "broken" + version = None + hashes = frozenset() + extras = frozenset() + + fake_result = _FakeResult( + mapping={("broken", frozenset()): _VersionlessCandidate()} + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"broken": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + # The versionless candidate was silently dropped — success + # with an EMPTY locked tuple rather than an exception. + assert isinstance(response.result, ResolverSuccess) + assert tuple(response.result.locked) == () + + def test_candidate_with_extras_and_multiple_hashes(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + + # Candidate carries two hashes + one extra — both should + # propagate into the LockedRequirement. + candidate_with_extras = Candidate( + name="django", + version="5.0.0", + url=f"{_INDEX}/django/django-5.0.0-py3-none-any.whl", + filename="django-5.0.0-py3-none-any.whl", + hashes=frozenset( + { + Hash("sha256", "a" * 64), + Hash("sha256", "b" * 64), + } + ), + requires_python=">=3.10", + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=True, + wheel_tags=None, + extras=frozenset({"argon2", "bcrypt"}), + ) + # T_S5: the translator looks up cache siblings at the resolved + # (name, version) and unions their hashes. Cache the + # candidate itself so the multi-hash assertion below survives + # the all-distfile-collection semantics (single artifact, two + # hashes — e.g. an index advertising md5 and sha256 for one + # wheel). + cache = _FakeCache( + {(_INDEX, "django"): (candidate_with_extras,)} + ) + fake_result = _FakeResult( + mapping={ + ("django", frozenset({"argon2", "bcrypt"})): + candidate_with_extras, + } + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"django": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + entry = locked[0] + # Hashes deduplicated & sorted; both ``a*64`` and ``b*64`` + # present. + assert len(entry.hashes) == 2 + assert all(h.startswith("sha256:") for h in entry.hashes) + assert tuple(entry.hashes) == tuple(sorted(entry.hashes)) + # Extras sorted alphabetically. + assert entry.extras == ("argon2", "bcrypt") + # T_M4: ``_build_request`` advertises ``Source(name="pypi", + # url=_INDEX)`` ⇒ the URL maps to the source NAME ``"pypi"`` + # (pip-parity); pre-T_M4 this was the URL verbatim. + assert entry.index == "pypi" + + def test_empty_specs_yields_empty_locked(self): + """A request with an empty ``packages.specs`` is a degenerate + but legal shape — the backend must not blow up and must + return :class:`ResolverSuccess` with an empty locked tuple. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + backend = PurePythonBackend( + cache=_FakeCache({}), + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({}) # no top-level packages + + # _drive_resolver returns an empty mapping for an empty + # requirement set in production; mock the same. + fake_result = _FakeResult(mapping={}) + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ) as drive: + response = backend.resolve(request) + + assert drive.called + assert isinstance(response.result, ResolverSuccess) + assert tuple(response.result.locked) == () + + +# ---------------------------------------------------------------------- +# T_S5 (Initiative G Phase 3b): _translate_mapping collects hashes from +# every distfile sibling at the resolved (name, version). +# ---------------------------------------------------------------------- +# +# Pip's lockfile convention emits hashes for ALL distfiles of the +# resolved version (wheel + sdist + any cross-platform wheel variants), +# not just the chosen candidate's single hash. The translator therefore +# walks the manifest cache across every configured index URL, picks out +# every cached ``Candidate`` whose ``version`` matches the resolved one, +# and unions their ``hashes`` frozensets. +# +# Fallback semantics: when the cache returns no sibling candidates +# (cold cache / test fixture that injects candidates straight into the +# result without populating the cache), the translator falls back to +# the resolved candidate's own ``hashes`` so Phase 3a-style fixtures +# keep working. +class TestTranslateMappingAllDistfileHashes: + """T_S5 — :meth:`PurePythonBackend._translate_mapping` collects + hashes from every distfile sibling at the resolved (name, version). + """ + + @staticmethod + def _wheel_with_hash(name: str, version: str, hash_value: str) -> Candidate: + filename = f"{name}-{version}-py3-none-any.whl" + return Candidate( + name=name, + version=version, + url=f"{_INDEX}/{name}/{filename}", + filename=filename, + hashes=frozenset({Hash("sha256", hash_value)}), + requires_python=None, + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=True, + wheel_tags=None, + extras=frozenset(), + ) + + @staticmethod + def _sdist_with_hash(name: str, version: str, hash_value: str) -> Candidate: + filename = f"{name}-{version}.tar.gz" + return Candidate( + name=name, + version=version, + url=f"{_INDEX}/{name}/{filename}", + filename=filename, + hashes=frozenset({Hash("sha256", hash_value)}), + requires_python=None, + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=False, + wheel_tags=None, + extras=frozenset(), + ) + + def test_collects_hashes_from_wheel_and_sdist_siblings(self): + """Cache holds two candidates at v1.0 (wheel + sdist); + resolved candidate is the wheel. Both hashes must appear. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + wheel_hash = "a" * 64 + sdist_hash = "b" * 64 + wheel = self._wheel_with_hash("foo", "1.0", wheel_hash) + sdist = self._sdist_with_hash("foo", "1.0", sdist_hash) + cache = _FakeCache({(_INDEX, "foo"): (wheel, sdist)}) + + fake_result = _FakeResult( + mapping={("foo", frozenset()): wheel} + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"foo": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + entry = locked[0] + # Both hashes propagated, sorted lexicographically. + assert tuple(entry.hashes) == ( + f"sha256:{wheel_hash}", + f"sha256:{sdist_hash}", + ) + + def test_only_matching_version_siblings_contribute(self): + """Cache holds candidates at v1.0 AND v2.0; resolved is v1.0. + Only v1.0 distfile hashes contribute. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + v1_wheel_hash = "1" * 64 + v1_sdist_hash = "2" * 64 + v2_wheel_hash = "3" * 64 + v1_wheel = self._wheel_with_hash("foo", "1.0", v1_wheel_hash) + v1_sdist = self._sdist_with_hash("foo", "1.0", v1_sdist_hash) + v2_wheel = self._wheel_with_hash("foo", "2.0", v2_wheel_hash) + cache = _FakeCache( + {(_INDEX, "foo"): (v1_wheel, v1_sdist, v2_wheel)} + ) + + fake_result = _FakeResult( + mapping={("foo", frozenset()): v1_wheel} + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"foo": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + entry = locked[0] + # v2 hash NEVER appears. + assert f"sha256:{v2_wheel_hash}" not in entry.hashes + # Both v1 hashes appear. + assert tuple(entry.hashes) == ( + f"sha256:{v1_wheel_hash}", + f"sha256:{v1_sdist_hash}", + ) + + def test_multi_index_dedup(self): + """Two index URLs both serve the same wheel ⇒ wheel hash + appears once in the lockfile (deduplicated by (algo, value)). + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + alt_index = "https://alt.example.com/simple" + wheel_hash = "c" * 64 + wheel_a = self._wheel_with_hash("foo", "1.0", wheel_hash) + wheel_b = self._wheel_with_hash("foo", "1.0", wheel_hash) + cache = _FakeCache( + { + (_INDEX, "foo"): (wheel_a,), + (alt_index, "foo"): (wheel_b,), + } + ) + + fake_result = _FakeResult( + mapping={("foo", frozenset()): wheel_a} + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + # Both indexes are configured on the backend so the + # translator scans both. + index_urls=(_INDEX, alt_index), + ) + # The request's sources only list ``_INDEX``; the backend's + # ``request_index_urls`` derivation prefers request.sources when + # populated. To exercise multi-index scanning we widen the + # request sources here. + request = ResolverRequest( + schema_version=SCHEMA_VERSION, + category="default", + packages=PackageSpecs(specs={"foo": "*"}), + options=ResolverOptions(), + sources=( + Source(name="pypi", url=_INDEX, verify_ssl=True), + Source(name="alt", url=alt_index, verify_ssl=True), + ), + ) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + entry = locked[0] + # Dedup: the same (sha256, hash_value) pair from both indexes + # collapses to one entry. + assert tuple(entry.hashes) == (f"sha256:{wheel_hash}",) + + def test_no_cache_falls_back_to_candidate_hashes(self): + """Cache returns ``None`` (cold cache / fixture-injected + candidate). Translator falls back to the resolved candidate's + own hashes — Phase 3a behaviour preserved as a fallback. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + wheel_hash = "d" * 64 + wheel = self._wheel_with_hash("foo", "1.0", wheel_hash) + # Cache is empty — every ``get`` returns None. + cache = _FakeCache({}) + + fake_result = _FakeResult( + mapping={("foo", frozenset()): wheel} + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + # Note: T_S4's emptiness pre-check fires when the top-level + # name has zero cached candidates across configured indexes — + # so we bypass the pre-check by handing the backend an EMPTY + # request packages.specs. The fake _drive_resolver mock then + # returns a candidate directly (fixture-injected workflow). + request = _build_request({}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + entry = locked[0] + # Fallback path emitted the candidate's own hash. + assert tuple(entry.hashes) == (f"sha256:{wheel_hash}",) + + def test_candidate_with_no_hashes_emits_empty_tuple(self): + """Resolved candidate has an empty ``hashes`` frozenset and the + cache has no sibling candidates ⇒ ``LockedRequirement.hashes`` + is an empty tuple (no crash, no synthesised data). + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + # Candidate with NO hashes (frozenset()). + hashless = Candidate( + name="foo", + version="1.0", + url=f"{_INDEX}/foo/foo-1.0-py3-none-any.whl", + filename="foo-1.0-py3-none-any.whl", + hashes=frozenset(), + requires_python=None, + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=True, + wheel_tags=None, + extras=frozenset(), + ) + cache = _FakeCache({}) + + fake_result = _FakeResult( + mapping={("foo", frozenset()): hashless} + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + # Empty specs ⇒ bypass T_S4 emptiness pre-check (see neighbouring + # test for the rationale). + request = _build_request({}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + entry = locked[0] + assert tuple(entry.hashes) == () + + +# ---------------------------------------------------------------------- +# T_M3 (Initiative G Phase 3b): _translate_mapping emits ``markers`` on +# LockedRequirement. +# ---------------------------------------------------------------------- +# +# The translator combines two marker sources: +# +# 1. **Requires-Python** — derived from ``candidate.requires_python`` +# (a ``SpecifierSet``-shaped string like ``">=3.10"``). Each spec is +# rendered as ``python_version ''`` and joined with +# ``and``. +# 2. **Introducing markers** — the ``introducing_marker`` slot +# populated by T_M2 on every transitive ``Requirement``. Pulled +# from ``Result.criteria[identifier].information`` (the +# ``RequirementInformation`` rows that selected this candidate). +# Multiple introducing markers OR-join with parentheses (set theory: +# a candidate satisfies the union of preconditions when any one +# requirement holds). +# +# Combined with ``and`` when both sources contribute; ``None`` when +# neither does. +class TestTranslateMappingMarkers: + """T_M3 marker emission on :meth:`PurePythonBackend._translate_mapping`.""" + + @staticmethod + def _candidate_with_requires_python(name: str, version: str, requires_python): + """Build a wheel :class:`Candidate` overriding ``requires_python``.""" + filename = f"{name}-{version}-py3-none-any.whl" + return Candidate( + name=name, + version=version, + url=f"{_INDEX}/{name}/{filename}", + filename=filename, + hashes=frozenset({Hash("sha256", "0" * 64)}), + requires_python=requires_python, + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=True, + wheel_tags=None, + extras=frozenset(), + ) + + @staticmethod + def _info_rows(*requirements): + """Build a tuple of ``RequirementInformation`` rows; parent is + irrelevant for T_M3 (only ``.requirement.introducing_marker`` is + consulted).""" + from pipenv.patched.pip._vendor.resolvelib.structs import ( + RequirementInformation, + ) + + return tuple( + RequirementInformation(requirement=r, parent=None) + for r in requirements + ) + + def _resolve_for(self, *, candidate, criteria_info, top_name="pkg"): + """Run :meth:`PurePythonBackend.resolve` with a synthetic + ``_drive_resolver`` return value carrying ``mapping`` + the + provided ``criteria_info``. Returns the single + :class:`LockedRequirement` emitted (asserts exactly one). + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + identifier = (top_name, frozenset()) + cache = _FakeCache({(_INDEX, top_name): (candidate,)}) + fake_result = _FakeResult( + mapping={identifier: candidate}, + criteria={identifier: _FakeCriterion(criteria_info)}, + ) + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({top_name: "*"}) + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + return locked[0] + + def test_requires_python_emits_marker(self): + """``requires_python=">=3.10"`` alone → markers carries the + translated ``python_version >= '3.10'`` clause.""" + cand = self._candidate_with_requires_python("click", "8.3.3", ">=3.10") + entry = self._resolve_for( + candidate=cand, + criteria_info=(), # no introducing markers + top_name="click", + ) + assert entry.markers == "python_version >= '3.10'" + + def test_requires_python_range_emits_combined_marker(self): + """``requires_python=">=3.8,<4"`` → ``python_version`` lower- + and upper-bound joined with ``and`` in stable sorted order. + """ + cand = self._candidate_with_requires_python("foo", "1.0", ">=3.8,<4") + entry = self._resolve_for( + candidate=cand, + criteria_info=(), + top_name="foo", + ) + # The translator sorts specs to keep cross-run output stable. + # ``"python_version < '4'"`` sorts before ``"python_version >= '3.8'"`` + # under default string comparison (``<`` (0x3c) < ``>`` (0x3e)). + assert entry.markers == "python_version < '4' and python_version >= '3.8'" + + def test_introducing_marker_alone_emits_marker(self): + """Candidate has no ``requires_python``; one ``Requirement`` + with a non-None ``introducing_marker`` selected it → its marker + string is the sole contribution.""" + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.markers import Marker + from pipenv.vendor.packaging.specifiers import SpecifierSet + + intro = Marker("python_version < '3.10'") + req = Requirement( + name="bar", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=None, + source="transitive", + parent="some-parent", + introducing_marker=intro, + ) + cand = self._candidate_with_requires_python("bar", "1.0", None) + entry = self._resolve_for( + candidate=cand, + criteria_info=self._info_rows(req), + top_name="bar", + ) + # The marker text is rendered by ``packaging.markers.Marker.__str__`` + # which uses double quotes around literal values. We accept the + # canonical packaging-rendered form as-is rather than normalising + # to single quotes — both are PEP 508 valid and semantically + # identical, and matching the upstream form keeps the translator + # zero-allocation on the hot path. + assert entry.markers == 'python_version < "3.10"' + + def test_requires_python_and_introducing_marker_combined(self): + """Both sources contribute → joined with ``and``.""" + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.markers import Marker + from pipenv.vendor.packaging.specifiers import SpecifierSet + + intro = Marker("python_version < '3.12'") + req = Requirement( + name="baz", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=None, + source="transitive", + parent="some-parent", + introducing_marker=intro, + ) + cand = self._candidate_with_requires_python("baz", "1.0", ">=3.8") + entry = self._resolve_for( + candidate=cand, + criteria_info=self._info_rows(req), + top_name="baz", + ) + # Combined with ``and`` (no parentheses for a single + # introducing marker — only the multi-intro form parenthesises). + # The Requires-Python contribution uses single-quoted literals + # (``repr(str)``) while the introducing-marker contribution + # comes through ``packaging.markers.Marker.__str__`` which + # uses double quotes. Two quoting conventions live side-by-side + # in the same output — both are PEP 508 valid and we deliberately + # preserve each source's canonical form rather than normalising. + assert entry.markers == ( + "python_version >= '3.8' and python_version < \"3.12\"" + ) + + def test_multiple_introducing_markers_or_joined(self): + """Two requirements with distinct ``introducing_marker``s selected + the same candidate → markers OR-joined with parentheses (a + candidate satisfies the *union* of its requirements).""" + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.markers import Marker + from pipenv.vendor.packaging.specifiers import SpecifierSet + + intro_a = Marker("python_version < '3.10'") + intro_b = Marker("python_version >= '3.12'") + req_a = Requirement( + name="multi", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=None, + source="transitive", + parent="parent-a", + introducing_marker=intro_a, + ) + req_b = Requirement( + name="multi", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=None, + source="transitive", + parent="parent-b", + introducing_marker=intro_b, + ) + cand = self._candidate_with_requires_python("multi", "1.0", None) + entry = self._resolve_for( + candidate=cand, + criteria_info=self._info_rows(req_a, req_b), + top_name="multi", + ) + assert entry.markers is not None + # Order-stable: insertion-order of ``information`` rows is + # preserved by the translator so a fixture-driven assertion is + # robust. Marker strings come through + # ``packaging.markers.Marker.__str__`` (double-quoted literals). + assert entry.markers == ( + '(python_version < "3.10") or (python_version >= "3.12")' + ) + + def test_no_requires_python_no_introducing_marker_emits_none(self): + """No ``requires_python`` AND no introducing marker → ``markers`` + stays ``None`` on the lockfile entry.""" + cand = self._candidate_with_requires_python("nada", "1.0", None) + entry = self._resolve_for( + candidate=cand, + criteria_info=(), + top_name="nada", + ) + assert entry.markers is None + + def test_invalid_requires_python_falls_back_to_none(self): + """An unparseable ``requires_python`` is treated as no + contribution (defensive — mirrors T4's behaviour around bad + ``Requires-Python`` advertisements on index manifests).""" + cand = self._candidate_with_requires_python( + "weirdpkg", "1.0", ">>not-a-spec<<" + ) + entry = self._resolve_for( + candidate=cand, + criteria_info=(), + top_name="weirdpkg", + ) + # The bad ``requires_python`` produced no marker; no introducing + # marker either → final ``markers`` is ``None``. + assert entry.markers is None + + +# ---------------------------------------------------------------------- +# T_M4 (Initiative G Phase 3b): _translate_mapping emits source NAME for +# the ``index`` field, not the URL. +# ---------------------------------------------------------------------- +# +# Pip's lockfile writes ``index=`` (e.g. ``"pypi"``) per the +# Pipfile ``[[source]]`` block's ``name`` key — NOT the raw URL. Pre-T_M4 +# the pure-python backend wrote ``index=`` because the Phase 3 +# Candidate doesn't track which configured source served it (T4 walks +# every ``index_url`` and concatenates). +# +# T_M4 closes the gap by building a ``url → name`` map from +# ``request.sources`` and looking up the source name at translate time. +# When the URL doesn't match any configured source (defensive fallback — +# Phase 4 will track per-candidate source attribution), the URL is +# emitted as-is. When ``request.sources`` is empty (subprocess fixtures +# sometimes omit it), the backend's configured default-URL is used +# directly (still a URL — no source list ⇒ no mapping possible). +class TestTranslateMappingIndexName: + """T_M4: ``LockedRequirement.index`` carries the source NAME when the + candidate's index-URL matches a configured ``[[source]]`` block; + otherwise the URL is emitted verbatim as a defensive fallback. + """ + + def test_url_maps_to_source_name(self): + """``request.sources = [Source(name="pypi", url="https://pypi.org/simple")]`` + + candidate from that URL ⇒ ``index == "pypi"`` (pip-parity).""" + from pipenv.resolver.backends.pure_python import PurePythonBackend + + cache = _FakeCache( + {(_INDEX, "click"): (_wheel_candidate("click", "8.3.3"),)} + ) + resolved = _wheel_candidate("click", "8.3.3") + fake_result = _FakeResult( + mapping={("click", frozenset()): resolved} + ) + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + # _build_request() already produces sources=(Source(name="pypi", + # url=_INDEX, verify_ssl=True),) — the canonical single-source + # case. + request = _build_request({"click": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + # Source NAME, not URL: pip writes ``"pypi"``, not the URL. + assert locked[0].index == "pypi" + + def test_unmatched_url_falls_back_to_url(self): + """``request.sources`` carries a single source whose URL does NOT + match the backend's configured ``index_urls`` ⇒ the URL is + emitted verbatim as a defensive fallback. + + Construction: pass ``index_urls`` explicitly so the resolve path + consults that URL while ``request.sources`` advertises a + different one. The URL→name map built from ``request.sources`` + therefore has no entry for the default URL ⇒ fallback fires. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + # Backend's default index — the URL the resolve path lands on. + unmatched_url = "https://pypi.org/simple" + # request.sources advertises a different URL under a custom name. + # The URL→name map built from ``request.sources`` will only + # contain ``https://other.example/simple -> "custom"`` — no entry + # for the unmatched_url default, so the fallback fires. + custom_source = Source( + name="custom", url="https://other.example/simple", verify_ssl=True + ) + cache = _FakeCache( + { + (unmatched_url, "pkg"): (_wheel_candidate("pkg", "1.0.0"),), + # Also seed the request-source URL so the prefetch path + # has something to find (not strictly required but + # mirrors a realistic production cache). + (custom_source.url, "pkg"): ( + _wheel_candidate("pkg", "1.0.0"), + ), + } + ) + resolved = _wheel_candidate("pkg", "1.0.0") + fake_result = _FakeResult( + mapping={("pkg", frozenset()): resolved} + ) + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + # Force the resolve path to land on unmatched_url as the + # default_index even though request.sources advertises + # custom_source.url. We achieve this by building a request + # whose sources contain ONLY custom_source — then + # request_index_urls is (custom_source.url,) and + # default_index is custom_source.url, which IS in url_to_name + # ⇒ wrong test. Reverse: supply request.sources with a + # source whose URL doesn't match the default_index — but + # default_index IS the first source URL. So the path can + # only produce an unmatched default_index when the URL→name + # map omits it. Concretely: the test exercises the + # defensive branch via the dict.get fallback semantic — see + # below. + ) + # Build a request whose sources advertise custom_source. The + # backend will set ``request_index_urls = (custom_source.url,)`` + # and ``default_index = custom_source.url``. ``url_to_name`` + # maps ``custom_source.url -> "custom"`` ⇒ this is the + # HAPPY-path mapping, not the fallback. + # + # To exercise the FALLBACK we instead force ``default_index`` to + # a URL absent from ``url_to_name`` by hand-patching the + # translator: we want a unit-level assertion on the defensive + # branch, so call ``_translate_mapping`` directly with a mocked + # url_to_name that omits the default_index. + from pipenv.resolver.schema import LockedRequirement + + url_to_name = {custom_source.url: custom_source.name} + # default_index = unmatched_url which is NOT in url_to_name. + criteria: dict = {} + from pipenv.resolver.backends.pure_python import PurePythonBackend as _B + + translator_backend = _B( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + + class _StubResult: + def __init__(self): + self.mapping = {("pkg", frozenset()): resolved} + self.criteria = criteria + + locked = translator_backend._translate_mapping( + _StubResult(), + (unmatched_url,), # index_urls[0] = unmatched_url + url_to_name, + ) + assert len(locked) == 1 + # Defensive fallback: no source matched ⇒ emit URL verbatim. + assert isinstance(locked[0], LockedRequirement) + assert locked[0].index == unmatched_url + + def test_no_sources_emits_url(self): + """``request.sources == ()`` (subprocess fixtures sometimes omit + the source list) ⇒ ``url_to_name`` is empty ⇒ defensive fallback + emits the default-index URL verbatim. No source list ⇒ no + mapping is possible. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + from pipenv.resolver.schema import ( + PackageSpecs, + ResolverOptions, + ResolverRequest, + ) + + cache = _FakeCache( + {(_INDEX, "lonely"): (_wheel_candidate("lonely", "1.0.0"),)} + ) + resolved = _wheel_candidate("lonely", "1.0.0") + fake_result = _FakeResult( + mapping={("lonely", frozenset()): resolved} + ) + # Build a request with an EMPTY sources tuple — the backend + # falls through to its configured default ``index_urls``. + request = ResolverRequest( + schema_version=SCHEMA_VERSION, + category="default", + packages=PackageSpecs(specs={"lonely": "*"}), + options=ResolverOptions(), + sources=(), + ) + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + index_urls=(_INDEX,), + ) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + # No sources ⇒ no URL→name mapping possible ⇒ emit URL. + assert locked[0].index == _INDEX + + +# ---------------------------------------------------------------------- +# T_M5 (Initiative G Phase 3b): belt-and-braces edge cases for marker / +# index / hash translation. Coverage on _translate_mapping is already at +# 96 % after T_M3 + T_M4 + T_S5; these tests close the scope by pinning +# tricky combinations that crossed the boundary between the three tracks +# (e.g. full combo of all three contributions, exotic Requires-Python +# operators, whitespace tolerance, and cross-pollination between +# resolved candidates). +# ---------------------------------------------------------------------- +class TestTranslateMappingT_M5Edges: + """T_M5 — combinational edge cases that exercise the + Requires-Python ↔ introducing-marker ↔ multi-distfile-hashes + intersections built up across T_M3, T_M4, and T_S5. + """ + + @staticmethod + def _wheel_with( + name: str, + version: str, + *, + requires_python: str | None = None, + hash_value: str = "0" * 64, + ) -> Candidate: + filename = f"{name}-{version}-py3-none-any.whl" + return Candidate( + name=name, + version=version, + url=f"{_INDEX}/{name}/{filename}", + filename=filename, + hashes=frozenset({Hash("sha256", hash_value)}), + requires_python=requires_python, + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=True, + wheel_tags=None, + extras=frozenset(), + ) + + @staticmethod + def _sdist_with( + name: str, + version: str, + *, + requires_python: str | None = None, + hash_value: str = "1" * 64, + ) -> Candidate: + filename = f"{name}-{version}.tar.gz" + return Candidate( + name=name, + version=version, + url=f"{_INDEX}/{name}/{filename}", + filename=filename, + hashes=frozenset({Hash("sha256", hash_value)}), + requires_python=requires_python, + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=False, + wheel_tags=None, + extras=frozenset(), + ) + + @staticmethod + def _info_rows(*requirements): + from pipenv.patched.pip._vendor.resolvelib.structs import ( + RequirementInformation, + ) + + return tuple( + RequirementInformation(requirement=r, parent=None) + for r in requirements + ) + + def test_full_combo_marker_requires_python_and_multi_distfile_hashes(self): + """All three contributions in one lockfile entry: + + * ``Requires-Python`` (``>=3.10``) → ``python_version >= '3.10'`` + * ``introducing_marker`` (``sys_platform == 'darwin'``) → propagates + * Multi-distfile hashes — wheel + sdist siblings both emit hashes. + + Pins that the three independent paths inside ``_translate_mapping`` + compose cleanly (T_M3's marker join + T_S5's hash union + T_M4's + source-name lookup). This is the "everything at once" smoke gate + — a regression in any one path surfaces here. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.markers import Marker + from pipenv.vendor.packaging.specifiers import SpecifierSet + + wheel_hash = "a" * 64 + sdist_hash = "b" * 64 + # Wheel chosen as the resolved candidate; sdist sibling at the + # same version contributes its hash to the lockfile entry. + wheel = self._wheel_with( + "combo", "1.0", requires_python=">=3.10", hash_value=wheel_hash + ) + sdist = self._sdist_with( + "combo", "1.0", requires_python=">=3.10", hash_value=sdist_hash + ) + cache = _FakeCache({(_INDEX, "combo"): (wheel, sdist)}) + + # Introducing requirement carries a marker — T_M2 contract. + intro_req = Requirement( + name="combo", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=None, + source="transitive", + parent="some-parent", + introducing_marker=Marker("sys_platform == 'darwin'"), + ) + + identifier = ("combo", frozenset()) + fake_result = _FakeResult( + mapping={identifier: wheel}, + criteria={identifier: _FakeCriterion(self._info_rows(intro_req))}, + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"combo": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked = tuple(response.result.locked) + assert len(locked) == 1 + entry = locked[0] + + # All three fields populated: + # 1. markers: AND-join of Requires-Python + introducing marker. + # Requires-Python uses single-quoted repr; ``Marker.__str__`` + # uses double-quoted form (matches T_M3's pin). + assert entry.markers == ( + "python_version >= '3.10' and sys_platform == \"darwin\"" + ) + # 2. hashes: wheel + sdist siblings unioned, sorted. + assert tuple(entry.hashes) == ( + f"sha256:{wheel_hash}", + f"sha256:{sdist_hash}", + ) + # 3. index: T_M4 source-name lookup from ``request.sources``. + assert entry.index == "pypi" + + def test_requires_python_not_equal_operator_emits_marker(self): + """``requires_python="!=3.11"`` → marker emits + ``python_version != '3.11'``. Pins the rare-but-real + ``!=`` operator path through ``_requires_python_to_marker``. + """ + from pipenv.resolver.backends.pure_python import ( + _requires_python_to_marker, + ) + + # Direct helper assertion — the translator just delegates to + # this function for the Requires-Python contribution. + assert _requires_python_to_marker("!=3.11") == "python_version != '3.11'" + + def test_requires_python_compatible_release_operator_emits_marker(self): + """``requires_python="~=3.10"`` (PEP 440 compatible-release) + → marker emits ``python_version ~= '3.10'``. + + We pin whatever the implementation does today: the helper renders + each ``Specifier`` verbatim as ``python_version ``, + so ``~=`` survives as-is. A future canonicaliser that expands + ``~=`` to a lower+upper bound pair would break this assertion — + which is intentional: that change has lockfile-shape implications + and warrants a deliberate gate flip. + """ + from pipenv.resolver.backends.pure_python import ( + _requires_python_to_marker, + ) + + assert _requires_python_to_marker("~=3.10") == "python_version ~= '3.10'" + + def test_requires_python_with_whitespace_emits_canonical(self): + """``requires_python=">= 3.10"`` (whitespace around the operator) + → marker emits the canonical ``python_version >= '3.10'``. + + ``SpecifierSet`` accepts whitespace inside an operator/version + spec and normalises on parse — the marker translator's output + must therefore not carry the user's stray whitespace through. + """ + from pipenv.resolver.backends.pure_python import ( + _requires_python_to_marker, + ) + + assert _requires_python_to_marker(">= 3.10") == "python_version >= '3.10'" + + def test_multiple_candidates_with_same_requires_python_no_cross_pollination( + self, + ): + """Two resolved candidates that happen to share the same + ``Requires-Python`` value each get their own marker, computed + independently. Pins that the translator does not accidentally + memoise / reuse the marker string across mapping entries + (which would create a cross-pollination bug if one candidate's + introducing marker leaked into another's). + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.markers import Marker + from pipenv.vendor.packaging.specifiers import SpecifierSet + + # Both candidates pin Requires-Python ``>=3.10`` — the marker + # contribution from that source is identical. The introducing + # marker is DIFFERENT per candidate so we can detect any leak. + cand_a = self._wheel_with("a-pkg", "1.0", requires_python=">=3.10") + cand_b = self._wheel_with("b-pkg", "2.0", requires_python=">=3.10") + cache = _FakeCache( + { + (_INDEX, "a-pkg"): (cand_a,), + (_INDEX, "b-pkg"): (cand_b,), + } + ) + + intro_a = Requirement( + name="a-pkg", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=None, + source="transitive", + parent="root", + introducing_marker=Marker("sys_platform == 'linux'"), + ) + intro_b = Requirement( + name="b-pkg", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=None, + source="transitive", + parent="root", + introducing_marker=Marker("sys_platform == 'win32'"), + ) + + id_a = ("a-pkg", frozenset()) + id_b = ("b-pkg", frozenset()) + fake_result = _FakeResult( + mapping={id_a: cand_a, id_b: cand_b}, + criteria={ + id_a: _FakeCriterion(self._info_rows(intro_a)), + id_b: _FakeCriterion(self._info_rows(intro_b)), + }, + ) + + backend = PurePythonBackend( + cache=cache, + fetcher=_FakeFetcher(), + session=mock.MagicMock(), + metadata_cache=mock.MagicMock(), + ) + request = _build_request({"a-pkg": "*", "b-pkg": "*"}) + + with mock.patch( + "pipenv.resolver.backends.pure_python._drive_resolver", + return_value=fake_result, + ): + response = backend.resolve(request) + + assert isinstance(response.result, ResolverSuccess) + locked_by_name = {l.name: l for l in response.result.locked} + # Each candidate's marker mentions its own platform — no + # cross-pollination. + marker_a = locked_by_name["a-pkg"].markers + marker_b = locked_by_name["b-pkg"].markers + assert marker_a is not None and marker_b is not None + assert "linux" in marker_a and "win32" not in marker_a + assert "win32" in marker_b and "linux" not in marker_b + # Both still carry the shared Requires-Python contribution. + assert "python_version >= '3.10'" in marker_a + assert "python_version >= '3.10'" in marker_b + + +# ---------------------------------------------------------------------- +# T10: Backend registration +# ---------------------------------------------------------------------- +# +# These tests verify the pure-python backend is wired into the +# Initiative F registry (``pipenv/resolver/backends/__init__.py``) under +# the name ``"pure-python"``. T9 built the class; T10 makes it +# discoverable via ``get_backend``. +class TestBackendRegistration: + def test_get_backend_returns_pure_python(self): + from pipenv.resolver.backends import get_backend + backend = get_backend("pure-python") + assert backend.name == "pure-python" + + def test_pure_python_backend_is_available(self): + from pipenv.resolver.backends import get_backend + backend = get_backend("pure-python") + assert backend.is_available() is True + + +# ---------------------------------------------------------------------- +# T9b: Bootstrap-from-request (2026-05-12) +# ---------------------------------------------------------------------- +# +# T10 made ``PurePythonBackend.__init__`` zero-arg-constructible by +# defaulting the four collaborators (``cache``, ``fetcher``, ``session``, +# ``metadata_cache``) to ``None``. That left a gap: the registry path +# (``get_backend("pure-python")``) lands inside ``resolve()`` with every +# collaborator set to ``None``, so ``self._fetcher.populate(targets)`` +# would NPE. T9b closes the gap by adding ``_bootstrap_from_request`` +# which constructs sensible defaults from the request envelope when +# fields are missing. These tests pin the bootstrap path; the existing +# kwarg-injection path is unaffected (regression-covered by the 24 +# tests above). +class TestBootstrapFromRequest: + """:meth:`PurePythonBackend._bootstrap_from_request` populates the + four collaborators from the :class:`ResolverRequest` when they are + ``None`` on ``self``. Pre-injected collaborators win — the + bootstrap is idempotent and never overwrites a non-None field. + """ + + def test_bootstrap_populates_missing_collaborators(self): + from pipenv.resolver.backends.pure_python import PurePythonBackend + from pipenv.resolver.fetcher import ParallelFetcher + from pipenv.resolver.manifest_cache import ParsedManifestCache + from pipenv.resolver.pure_python_metadata import MetadataCache + + backend = PurePythonBackend() # zero-arg path (registry shape) + # Every collaborator starts as ``None`` — pins the precondition + # T10 introduced (without it, the bootstrap test would be + # vacuously true). + assert backend._cache is None + assert backend._fetcher is None + assert backend._session is None + assert backend._metadata_cache is None + + request = _build_request({"requests": "*"}) + backend._bootstrap_from_request(request) + + # All four collaborators are now populated with concrete + # instances of the canonical classes. + assert isinstance(backend._cache, ParsedManifestCache) + assert isinstance(backend._fetcher, ParallelFetcher) + assert backend._session is not None + assert isinstance(backend._metadata_cache, MetadataCache) + + # Idempotency: a second call must not rebuild any collaborator + # (sentinel-by-identity on each of the four). + prior_cache = backend._cache + prior_fetcher = backend._fetcher + prior_session = backend._session + prior_metadata_cache = backend._metadata_cache + backend._bootstrap_from_request(request) + assert backend._cache is prior_cache + assert backend._fetcher is prior_fetcher + assert backend._session is prior_session + assert backend._metadata_cache is prior_metadata_cache + + def test_bootstrap_preserves_injected_collaborators(self): + """Tests that pass collaborators via ``__init__`` kwargs must + not have them silently replaced by the bootstrap step. Pinned + explicitly because the regression vector here is invisible — + a future "always rebuild" refactor would pass every other test + in this file while breaking the kwarg-injection contract that + the rest of the suite (24 tests) relies on. + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + sentinel_cache = _FakeCache({}) + sentinel_fetcher = _FakeFetcher() + sentinel_session = mock.MagicMock(name="session-sentinel") + sentinel_metadata_cache = mock.MagicMock(name="metadata-cache-sentinel") + + backend = PurePythonBackend( + cache=sentinel_cache, + fetcher=sentinel_fetcher, + session=sentinel_session, + metadata_cache=sentinel_metadata_cache, + ) + backend._bootstrap_from_request(_build_request({"requests": "*"})) + + # Every pre-injected collaborator survived unchanged (identity + # check — same object, not "equal"). + assert backend._cache is sentinel_cache + assert backend._fetcher is sentinel_fetcher + assert backend._session is sentinel_session + assert backend._metadata_cache is sentinel_metadata_cache + + def test_bootstrap_falls_back_to_poolmanager_when_session_factory_fails( + self, monkeypatch + ): + """If ``get_requests_session`` import explodes, bootstrap must + fall back to a bare ``urllib3.PoolManager`` rather than crash. + Pinned because this last-resort branch only fires in unusual + sandboxed test paths (T_S6 — defensive bootstrap edge). + """ + import sys + + from pipenv.resolver.backends.pure_python import PurePythonBackend + from pipenv.patched.pip._vendor.urllib3 import PoolManager + + # Force the canonical-factory import to fail by shadowing the + # parent module with a broken stub that raises at attribute time. + broken_module = mock.MagicMock() + broken_module.get_requests_session = mock.MagicMock( + side_effect=ImportError("simulated sandbox: factory unavailable") + ) + monkeypatch.setitem(sys.modules, "pipenv.utils.internet", broken_module) + + backend = PurePythonBackend() + request = _build_request({"requests": "*"}) + backend._bootstrap_from_request(request) + + assert isinstance(backend._session, PoolManager) + + def test_cache_dir_honours_pipenv_cache_dir_env_var( + self, monkeypatch, tmp_path + ): + """``_cache_dir_from_request`` reads ``PIPENV_CACHE_DIR`` from + the environment when set (T_S6 — bootstrap cache root edge). + """ + from pipenv.resolver.backends.pure_python import PurePythonBackend + + custom = tmp_path / "custom-cache" + monkeypatch.setenv("PIPENV_CACHE_DIR", str(custom)) + + request = _build_request({"requests": "*"}) + resolved = PurePythonBackend._cache_dir_from_request(request) + assert resolved == custom + + def test_cache_dir_default_when_env_var_unset(self, monkeypatch): + """Without ``PIPENV_CACHE_DIR``, default path is + ``~/.cache/pipenv`` (T_S6 — bootstrap cache root edge). + """ + from pathlib import Path as _Path + + from pipenv.resolver.backends.pure_python import PurePythonBackend + + monkeypatch.delenv("PIPENV_CACHE_DIR", raising=False) + + request = _build_request({"requests": "*"}) + resolved = PurePythonBackend._cache_dir_from_request(request) + assert resolved == _Path.home() / ".cache" / "pipenv" diff --git a/tests/unit/test_pure_python_metadata.py b/tests/unit/test_pure_python_metadata.py new file mode 100644 index 0000000000..679bccb56c --- /dev/null +++ b/tests/unit/test_pure_python_metadata.py @@ -0,0 +1,1854 @@ +"""Unit tests for :mod:`pipenv.resolver.pure_python_metadata` +(Initiative G Phase 3, T2). + +This file is the RED-phase test suite that pins T2's contract. T12 +extends this file later with the full coverage matrix; the minimum +acceptance gate from the plan brief is: + +* PEP 658 fast path with mocked HTTP layer. +* Wheel-head fallback with a ``tmp_path``-built synthetic wheel. +* Cache round-trip. +* Hash-mismatch raises ``MetadataFetchError``. + +All tests use a duck-typed session ``MagicMock`` matching the +``urllib3.PoolManager.request`` shape that the rest of +``pipenv.resolver.*`` already uses (see ``pep691.py``). The fetcher +should call ``session.request("GET", url, headers=, ...)`` or +``session.request("HEAD", url, headers=, ...)``. +""" + +from __future__ import annotations + +import hashlib +import zipfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from pipenv.resolver.candidate import Candidate + +# Real-world METADATA body (truncated to relevant headers). Lifted from +# numpy 1.26.0's wheel METADATA. Lines retain the LF line ending that +# email.parser expects; double-LF ends the header block. +NUMPY_METADATA_TEXT = ( + "Metadata-Version: 2.1\n" + "Name: numpy\n" + "Version: 1.26.0\n" + "Summary: Fundamental package for array computing in Python\n" + "Home-page: https://numpy.org\n" + "Requires-Python: <3.13,>=3.9\n" + "Provides-Extra: test\n" + "Requires-Dist: hypothesis ; extra == 'test'\n" + "Requires-Dist: pytest ; extra == 'test'\n" + "Requires-Dist: pytest-cov ; extra == 'test'\n" + "\n" + "NumPy is the fundamental package for...\n" +) + + +def _make_wheel_candidate( + name: str = "numpy", + version: str = "1.26.0", + *, + url: str = "https://example.org/wheels/numpy-1.26.0-py3-none-any.whl", +) -> Candidate: + filename = url.rsplit("/", 1)[-1] + return Candidate.from_filename( + filename, + name=name, + version=version, + url=url, + hashes=frozenset(), + requires_python=">=3.9", + yanked=False, + yanked_reason=None, + upload_time=None, + ) + + +def _make_response( + *, + status: int = 200, + data: bytes = b"", + headers: dict[str, str] | None = None, +) -> MagicMock: + """Build a urllib3-response-shaped mock.""" + response = MagicMock() + response.status = status + response.data = data + response.headers = headers if headers is not None else {} + response.release_conn = MagicMock(return_value=None) + return response + + +def _make_session_router(routes): + """Build a session that picks a response per (method, url) lookup. + + ``routes`` is a list of ``(method, url_substring, response)`` tuples + consumed in order; the first matching tuple's response is returned. + A no-match raises so tests fail loudly instead of returning an + auto-MagicMock. + """ + + session = MagicMock() + call_log: list[tuple[str, str, dict]] = [] + + def _dispatch(method, url, *, headers=None, **_kwargs): + call_log.append((method, url, dict(headers or {}))) + for r_method, r_substr, response in routes: + if r_method == method and r_substr in url: + return response + raise AssertionError( + f"unexpected session call: {method} {url} " + f"(routes={[(m, s) for (m, s, _) in routes]})" + ) + + session.request.side_effect = _dispatch + session._call_log = call_log # type: ignore[attr-defined] + return session + + +# --------------------------------------------------------------------------- +# PEP 658 fast path +# --------------------------------------------------------------------------- + + +class TestPEP658FastPath: + """Fetch via ``.metadata`` when the index advertises it.""" + + def test_pep658_fast_path_returns_parsed_metadata(self): + from pipenv.resolver.pure_python_metadata import ( + CoreMetadata, + fetch_metadata, + ) + + body = NUMPY_METADATA_TEXT.encode("utf-8") + body_hash = hashlib.sha256(body).hexdigest() + + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + response = _make_response( + status=200, + data=body, + headers={"Content-Type": "text/plain"}, + ) + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + result = fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + metadata_hash={"sha256": body_hash}, + ) + + assert isinstance(result, CoreMetadata) + assert result.name == "numpy" + assert result.version == "1.26.0" + assert result.requires_python == "<3.13,>=3.9" + # Three Requires-Dist lines, in order. + assert "hypothesis ; extra == 'test'" in result.requires_dist + assert "pytest ; extra == 'test'" in result.requires_dist + assert "pytest-cov ; extra == 'test'" in result.requires_dist + assert len(result.requires_dist) == 3 + assert "test" in result.provides_extras + assert result.summary == ( + "Fundamental package for array computing in Python" + ) + + def test_pep658_hash_mismatch_raises(self): + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + body = NUMPY_METADATA_TEXT.encode("utf-8") + # Wrong hash on purpose. + wrong_hash = "0" * 64 + + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + response = _make_response( + status=200, + data=body, + headers={"Content-Type": "text/plain"}, + ) + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + with pytest.raises(MetadataFetchError) as excinfo: + fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + metadata_hash={"sha256": wrong_hash}, + ) + + # Surfacing should name the algo + the offender so users can + # audit which wheel poisoned the cache. + msg = str(excinfo.value) + assert "sha256" in msg.lower() + + +# --------------------------------------------------------------------------- +# Wheel-head fallback +# --------------------------------------------------------------------------- + + +def _build_synthetic_wheel(tmp_path: Path, metadata_text: str) -> bytes: + """Build a single-METADATA-member wheel bytes blob and return it.""" + wheel_path = tmp_path / "fakepkg-1.0.0-py3-none-any.whl" + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("fakepkg-1.0.0.dist-info/METADATA", metadata_text) + zf.writestr("fakepkg/__init__.py", "") + return wheel_path.read_bytes() + + +class TestWheelHeadFallback: + """Range-GET on the wheel zip's central directory; parse METADATA.""" + + def test_wheel_head_fallback_returns_parsed_metadata(self, tmp_path): + from pipenv.resolver.pure_python_metadata import fetch_metadata + + metadata_text = ( + "Metadata-Version: 2.1\n" + "Name: fakepkg\n" + "Version: 1.0.0\n" + "Summary: A synthetic wheel for tests\n" + "Requires-Python: >=3.9\n" + "Requires-Dist: requests>=2.0\n" + "Requires-Dist: click\n" + "\n" + ) + wheel_bytes = _build_synthetic_wheel(tmp_path, metadata_text) + + candidate = _make_wheel_candidate( + name="fakepkg", + version="1.0.0", + url=( + "https://example.org/wheels/" + "fakepkg-1.0.0-py3-none-any.whl" + ), + ) + + # HEAD reveals Content-Length; GET with a range returns the + # last 64 kB (whole wheel for a small synthetic). + head_response = _make_response( + status=200, + data=b"", + headers={"Content-Length": str(len(wheel_bytes))}, + ) + # The fetcher will issue at least one range GET; we serve the + # whole wheel for any GET to keep the mock simple. Real + # behaviour serves a 206 with a slice; the fetcher should + # accept 200 too (some mirrors don't honour Range). + get_response = _make_response( + status=206, + data=wheel_bytes, + headers={ + "Content-Range": ( + f"bytes 0-{len(wheel_bytes) - 1}/{len(wheel_bytes)}" + ), + "Content-Length": str(len(wheel_bytes)), + }, + ) + session = _make_session_router( + [ + ("HEAD", candidate.url, head_response), + ("GET", candidate.url, get_response), + ], + ) + + result = fetch_metadata(candidate, session) + + assert result.name == "fakepkg" + assert result.version == "1.0.0" + assert result.requires_python == ">=3.9" + assert "requests>=2.0" in result.requires_dist + assert "click" in result.requires_dist + assert result.summary == "A synthetic wheel for tests" + + def test_wheel_head_405_falls_back_to_probing_get(self, tmp_path): + """HEAD 405 → probing GET with ``Range: bytes=0-1`` for length.""" + + from pipenv.resolver.pure_python_metadata import fetch_metadata + + metadata_text = ( + "Metadata-Version: 2.1\n" + "Name: fakepkg\n" + "Version: 1.0.0\n" + "Requires-Dist: six\n" + "\n" + ) + wheel_bytes = _build_synthetic_wheel(tmp_path, metadata_text) + + candidate = _make_wheel_candidate( + name="fakepkg", + version="1.0.0", + url=( + "https://example.org/wheels/" + "fakepkg-1.0.0-py3-none-any.whl" + ), + ) + + # HEAD → 405 (not allowed). GET with a small range → 206 with + # Content-Range exposing the full length. Subsequent range GET + # returns the whole body. + head_response = _make_response(status=405, data=b"", headers={}) + probe_response = _make_response( + status=206, + data=wheel_bytes[:2], + headers={ + "Content-Range": f"bytes 0-1/{len(wheel_bytes)}", + "Content-Length": "2", + }, + ) + # We do GET twice (probe + actual range). The router returns + # the same response for both; the response data path is the + # one that mattered for probing (Content-Range), then for the + # real fetch we hand back the whole wheel as a permissive 200. + full_response = _make_response( + status=200, + data=wheel_bytes, + headers={"Content-Length": str(len(wheel_bytes))}, + ) + + # Order of routes matters: first GET → probe (small body), then + # we want a route for the real range GET that serves the full + # wheel. Encode that with a stateful side_effect. + get_calls = {"count": 0} + + def _dispatch(method, url, *, headers=None, **_kw): + if method == "HEAD": + return head_response + if method == "GET": + # First GET is the probe (Range: bytes=0-1). Subsequent + # GETs return the full body. + get_calls["count"] += 1 + if get_calls["count"] == 1: + return probe_response + return full_response + raise AssertionError( + f"unexpected session call: {method} {url}" + ) + + session = MagicMock() + session.request.side_effect = _dispatch + + result = fetch_metadata(candidate, session) + + assert result.name == "fakepkg" + assert "six" in result.requires_dist + + +# --------------------------------------------------------------------------- +# On-disk cache round-trip +# --------------------------------------------------------------------------- + + +class TestMetadataCache: + """A second fetch on a populated cache must not hit the network.""" + + def test_cache_round_trip_skips_network_on_second_call( + self, tmp_path + ): + from pipenv.resolver.pure_python_metadata import ( + MetadataCache, + fetch_metadata, + ) + + body = NUMPY_METADATA_TEXT.encode("utf-8") + body_hash = hashlib.sha256(body).hexdigest() + + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + response = _make_response( + status=200, + data=body, + headers={"Content-Type": "text/plain"}, + ) + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + cache = MetadataCache(tmp_path / "metadata-cache") + + result1 = fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + metadata_hash={"sha256": body_hash}, + cache=cache, + ) + assert result1.name == "numpy" + # One network call for the PEP 658 body. + assert session.request.call_count == 1 + + # Second call: cache hit; no further network. + result2 = fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + metadata_hash={"sha256": body_hash}, + cache=cache, + ) + assert result2.name == "numpy" + assert result2.requires_dist == result1.requires_dist + assert session.request.call_count == 1 # unchanged + + def test_cache_get_returns_none_when_missing(self, tmp_path): + from pipenv.resolver.pure_python_metadata import MetadataCache + + cache = MetadataCache(tmp_path / "metadata-cache") + assert cache.get("https://example.org/nonexistent.whl") is None + + def test_cache_put_then_get_round_trip(self, tmp_path): + from pipenv.resolver.pure_python_metadata import ( + CoreMetadata, + MetadataCache, + ) + + cache = MetadataCache(tmp_path / "metadata-cache") + meta = CoreMetadata( + name="six", + version="1.16.0", + requires_python=">=2.7", + requires_dist=("attrs>=21.0",), + provides_extras=frozenset({"test"}), + summary="Python 2 and 3 compatibility utilities", + ) + url = "https://example.org/wheels/six-1.16.0-py2.py3-none-any.whl" + cache.put(url, meta) + + out = cache.get(url) + assert out == meta + + +# --------------------------------------------------------------------------- +# _parse_metadata_text — the helper unit tests +# --------------------------------------------------------------------------- + + +class TestParseMetadataText: + """``_parse_metadata_text`` is the email-parser-fronted helper.""" + + def test_parses_minimum_required_fields(self): + from pipenv.resolver.pure_python_metadata import ( + _parse_metadata_text, + ) + + text = ( + "Metadata-Version: 2.1\n" + "Name: SomePkg\n" + "Version: 0.1.0\n" + "\n" + ) + result = _parse_metadata_text(text) + # Name should be PEP 503 canonical (lower-case here it already is). + assert result.name == "somepkg" + assert result.version == "0.1.0" + assert result.requires_python is None + assert result.requires_dist == () + assert result.provides_extras == frozenset() + + def test_collects_repeated_requires_dist(self): + from pipenv.resolver.pure_python_metadata import ( + _parse_metadata_text, + ) + + text = ( + "Metadata-Version: 2.1\n" + "Name: alpha\n" + "Version: 1.0\n" + "Requires-Dist: a>=1\n" + "Requires-Dist: b<2\n" + "Requires-Dist: c\n" + "Provides-Extra: dev\n" + "Provides-Extra: test\n" + "\n" + ) + result = _parse_metadata_text(text) + assert result.requires_dist == ("a>=1", "b<2", "c") + assert result.provides_extras == frozenset({"dev", "test"}) + + def test_populates_all_core_metadata_fields(self): + """``_parse_metadata_text`` populates every :class:`CoreMetadata` field. + + Covers the "full happy path" path through the email parser, + including multiple ``Provides-Extra`` lines, multiple + ``Requires-Dist`` lines (mixed with extras markers), a + non-canonical ``Name``, and a stripped ``Requires-Python``. + """ + from pipenv.resolver.pure_python_metadata import ( + _parse_metadata_text, + ) + + text = ( + "Metadata-Version: 2.1\n" + "Name: My_Funky.Pkg\n" + "Version: 0.2.0\n" + "Summary: Bench package \n" + "Requires-Python: >=3.8,<4 \n" + "Requires-Dist: a>=1\n" + "Requires-Dist: b ; extra == 'docs'\n" + "Requires-Dist: c ; extra == 'test'\n" + "Provides-Extra: docs\n" + "Provides-Extra: test\n" + "Provides-Extra: extras-with-dash\n" + "\n" + ) + result = _parse_metadata_text(text) + assert result.name == "my-funky-pkg" + assert result.version == "0.2.0" + assert result.summary == "Bench package" + assert result.requires_python == ">=3.8,<4" + assert result.requires_dist == ( + "a>=1", + "b ; extra == 'docs'", + "c ; extra == 'test'", + ) + assert result.provides_extras == frozenset( + {"docs", "test", "extras-with-dash"} + ) + + def test_blank_requires_python_normalises_to_none(self): + from pipenv.resolver.pure_python_metadata import ( + _parse_metadata_text, + ) + + text = ( + "Metadata-Version: 2.1\n" + "Name: blanky\n" + "Version: 1.0\n" + "Requires-Python: \n" + "Summary: \n" + "\n" + ) + result = _parse_metadata_text(text) + assert result.requires_python is None + assert result.summary is None + + +# --------------------------------------------------------------------------- +# CoreMetadata dataclass behaviour (frozen + slots) +# --------------------------------------------------------------------------- + + +class TestCoreMetadataDataclass: + """``CoreMetadata`` is ``frozen=True, slots=True`` — both must hold.""" + + def _make(self): + from pipenv.resolver.pure_python_metadata import CoreMetadata + + return CoreMetadata( + name="six", + version="1.16.0", + requires_python=">=2.7", + requires_dist=("attrs>=21.0",), + provides_extras=frozenset({"test"}), + summary="Python 2 and 3 compatibility utilities", + ) + + def test_is_frozen(self): + from dataclasses import FrozenInstanceError + + meta = self._make() + with pytest.raises(FrozenInstanceError): + meta.name = "seven" # type: ignore[misc] + + def test_uses_slots_no_dict(self): + meta = self._make() + # slots=True suppresses __dict__ allocation. + assert not hasattr(meta, "__dict__") + + def test_is_hashable_and_lives_in_frozenset(self): + meta_a = self._make() + meta_b = self._make() + bag = frozenset({meta_a, meta_b}) + assert len(bag) == 1 + assert meta_a in bag + + def test_equality_by_value(self): + assert self._make() == self._make() + + +# --------------------------------------------------------------------------- +# PEP 658 fast path — additional coverage +# --------------------------------------------------------------------------- + + +class TestPEP658Extra: + """Edge cases on the PEP 658 path that the T2 happy path missed.""" + + def test_pep658_empty_hash_dict_skips_verification(self): + """An empty ``metadata_hash`` dict must not raise; PEP 658 §6 allows it.""" + from pipenv.resolver.pure_python_metadata import fetch_metadata + + body = NUMPY_METADATA_TEXT.encode("utf-8") + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + response = _make_response(status=200, data=body) + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + result = fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + metadata_hash={}, + ) + assert result.name == "numpy" + + def test_pep658_hash_dict_without_sha256_skips_verification(self): + """A ``metadata_hash`` with ``md5`` but no ``sha256`` is treated as no-hash.""" + from pipenv.resolver.pure_python_metadata import fetch_metadata + + body = NUMPY_METADATA_TEXT.encode("utf-8") + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + response = _make_response(status=200, data=body) + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + # Only md5 advertised — we should NOT raise (matches T2's + # design note: "only sha256 is honoured; other algos are + # ignored"). + result = fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + metadata_hash={"md5": "deadbeef" * 4}, + ) + assert result.name == "numpy" + + def test_pep658_none_hash_skips_verification(self): + from pipenv.resolver.pure_python_metadata import fetch_metadata + + body = NUMPY_METADATA_TEXT.encode("utf-8") + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + response = _make_response(status=200, data=body) + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + result = fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + metadata_hash=None, + ) + assert result.name == "numpy" + + def test_pep658_non_utf8_body_raises(self): + """A non-UTF-8 PEP 658 body surfaces as ``MetadataFetchError``.""" + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + # Invalid UTF-8: a lone continuation byte. + body = b"\xff\xfe not utf-8 \x80\x81" + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + response = _make_response(status=200, data=body) + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + with pytest.raises(MetadataFetchError, match="not UTF-8"): + fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + ) + + def test_pep658_http_error_raises(self): + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + response = _make_response(status=404, data=b"") + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + with pytest.raises(MetadataFetchError, match="HTTP 404"): + fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + ) + + def test_pep658_session_raises_surfaces_as_metadata_fetch_error(self): + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + session = MagicMock() + session.request.side_effect = RuntimeError("network down") + + with pytest.raises(MetadataFetchError, match="no response"): + fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + ) + + def test_pep658_response_with_no_body_raises(self): + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + # data=None — some mocks/HTTP clients return this on a streamed body. + response = _make_response(status=200) + response.data = None + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + with pytest.raises(MetadataFetchError, match="no body"): + fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + ) + + +# --------------------------------------------------------------------------- +# Wheel-head fallback — additional coverage +# --------------------------------------------------------------------------- + + +def _build_wheel_without_metadata(tmp_path: Path) -> bytes: + """Build a wheel-shaped zip with NO ``/METADATA`` member.""" + wheel_path = tmp_path / "broken-1.0.0-py3-none-any.whl" + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf: + # Has a dist-info directory but lacks the METADATA file. + zf.writestr("broken-1.0.0.dist-info/RECORD", "") + zf.writestr("broken/__init__.py", "") + return wheel_path.read_bytes() + + +class TestWheelHeadFallbackExtra: + """Extra branches of the wheel-tail range-fetch path.""" + + def _candidate(self): + return _make_wheel_candidate( + name="broken", + version="1.0.0", + url=( + "https://example.org/wheels/" + "broken-1.0.0-py3-none-any.whl" + ), + ) + + def test_missing_metadata_member_raises(self, tmp_path): + """A wheel whose central directory has no ``METADATA`` member raises.""" + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + wheel_bytes = _build_wheel_without_metadata(tmp_path) + candidate = self._candidate() + + head_response = _make_response( + status=200, + headers={"Content-Length": str(len(wheel_bytes))}, + ) + get_response = _make_response( + status=206, + data=wheel_bytes, + headers={ + "Content-Range": ( + f"bytes 0-{len(wheel_bytes) - 1}/{len(wheel_bytes)}" + ), + }, + ) + session = _make_session_router( + [ + ("HEAD", candidate.url, head_response), + ("GET", candidate.url, get_response), + ], + ) + + with pytest.raises(MetadataFetchError, match="METADATA"): + fetch_metadata(candidate, session) + + def test_corrupt_zip_tail_raises(self, tmp_path): + """If the tail bytes can't be parsed as a zip, we surface an error.""" + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + candidate = self._candidate() + # Junk that is neither a zip nor anywhere near the central directory. + junk = b"\x00" * 1024 + head_response = _make_response( + status=200, + headers={"Content-Length": str(len(junk))}, + ) + get_response = _make_response( + status=206, + data=junk, + headers={ + "Content-Range": f"bytes 0-{len(junk) - 1}/{len(junk)}", + }, + ) + session = _make_session_router( + [ + ("HEAD", candidate.url, head_response), + ("GET", candidate.url, get_response), + ], + ) + + with pytest.raises(MetadataFetchError, match="central directory"): + fetch_metadata(candidate, session) + + def test_head_405_no_content_range_raises(self): + """HEAD 405 + probing GET 206 with no Content-Range header → error.""" + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + candidate = self._candidate() + head_response = _make_response(status=405) + probe_response = _make_response( + status=206, + data=b"ab", + headers={"Content-Length": "2"}, # no Content-Range + ) + session = _make_session_router( + [ + ("HEAD", candidate.url, head_response), + ("GET", candidate.url, probe_response), + ], + ) + + with pytest.raises( + MetadataFetchError, match="could not parse Content-Range" + ): + fetch_metadata(candidate, session) + + def test_head_405_probe_get_4xx_raises(self): + """HEAD 405 + probing GET 403 → error mentioning the status.""" + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + candidate = self._candidate() + head_response = _make_response(status=405) + probe_response = _make_response(status=403) + session = _make_session_router( + [ + ("HEAD", candidate.url, head_response), + ("GET", candidate.url, probe_response), + ], + ) + + with pytest.raises(MetadataFetchError, match="HTTP 403"): + fetch_metadata(candidate, session) + + def test_head_and_probe_both_session_failure_raises(self): + """Both HEAD and probing GET raising at the session layer → error.""" + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + candidate = self._candidate() + session = MagicMock() + session.request.side_effect = RuntimeError("network gone") + + with pytest.raises( + MetadataFetchError, match="HEAD and probing GET both failed" + ): + fetch_metadata(candidate, session) + + def test_head_200_no_content_length_falls_back_to_probe(self, tmp_path): + """HEAD 200 without a ``Content-Length`` header falls through to the probe.""" + from pipenv.resolver.pure_python_metadata import fetch_metadata + + metadata_text = ( + "Metadata-Version: 2.1\n" + "Name: tinypkg\n" + "Version: 0.1.0\n" + "Requires-Dist: x\n" + "\n" + ) + wheel_bytes = _build_synthetic_wheel(tmp_path, metadata_text) + + candidate = _make_wheel_candidate( + name="tinypkg", + version="0.1.0", + url=( + "https://example.org/wheels/" + "tinypkg-0.1.0-py3-none-any.whl" + ), + ) + + # HEAD: 200 with no Content-Length (forces the fallback path). + head_response = _make_response(status=200, headers={}) + # First GET (probe with bytes=0-1) → 206 with a usable Content-Range. + probe_response = _make_response( + status=206, + data=wheel_bytes[:2], + headers={ + "Content-Range": f"bytes 0-1/{len(wheel_bytes)}", + "Content-Length": "2", + }, + ) + # Second GET (the real range fetch) → 206 carrying the wheel. + full_response = _make_response( + status=206, + data=wheel_bytes, + headers={ + "Content-Range": ( + f"bytes 0-{len(wheel_bytes) - 1}/{len(wheel_bytes)}" + ), + }, + ) + + get_calls = {"count": 0} + + def _dispatch(method, url, *, headers=None, **_kw): + if method == "HEAD": + return head_response + if method == "GET": + get_calls["count"] += 1 + if get_calls["count"] == 1: + return probe_response + return full_response + raise AssertionError(f"unexpected: {method} {url}") + + session = MagicMock() + session.request.side_effect = _dispatch + + result = fetch_metadata(candidate, session) + assert result.name == "tinypkg" + + def test_head_200_bad_content_length_falls_back(self, tmp_path): + """A non-integer ``Content-Length`` on HEAD is treated as missing.""" + from pipenv.resolver.pure_python_metadata import fetch_metadata + + metadata_text = ( + "Metadata-Version: 2.1\n" + "Name: lengthbad\n" + "Version: 0.1.0\n" + "Requires-Dist: y\n" + "\n" + ) + wheel_bytes = _build_synthetic_wheel(tmp_path, metadata_text) + candidate = _make_wheel_candidate( + name="lengthbad", + version="0.1.0", + url=( + "https://example.org/wheels/" + "lengthbad-0.1.0-py3-none-any.whl" + ), + ) + + head_response = _make_response( + status=200, + headers={"Content-Length": "not-an-int"}, + ) + probe_response = _make_response( + status=206, + data=wheel_bytes[:2], + headers={ + "Content-Range": f"bytes 0-1/{len(wheel_bytes)}", + }, + ) + full_response = _make_response( + status=206, + data=wheel_bytes, + headers={ + "Content-Range": ( + f"bytes 0-{len(wheel_bytes) - 1}/{len(wheel_bytes)}" + ), + }, + ) + + get_calls = {"count": 0} + + def _dispatch(method, url, *, headers=None, **_kw): + if method == "HEAD": + return head_response + get_calls["count"] += 1 + if get_calls["count"] == 1: + return probe_response + return full_response + + session = MagicMock() + session.request.side_effect = _dispatch + + result = fetch_metadata(candidate, session) + assert result.name == "lengthbad" + + def test_head_405_content_range_asterisk_total_raises(self): + """Some servers return ``bytes 0-1/*`` — unparseable → error.""" + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + candidate = self._candidate() + head_response = _make_response(status=405) + probe_response = _make_response( + status=206, + data=b"ab", + headers={"Content-Range": "bytes 0-1/*"}, + ) + session = _make_session_router( + [ + ("HEAD", candidate.url, head_response), + ("GET", candidate.url, probe_response), + ], + ) + + with pytest.raises(MetadataFetchError, match="could not parse Content-Range"): + fetch_metadata(candidate, session) + + def test_head_405_unparseable_total_raises(self): + """Garbage after the slash in ``Content-Range`` → error.""" + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + fetch_metadata, + ) + + candidate = self._candidate() + head_response = _make_response(status=405) + probe_response = _make_response( + status=206, + data=b"ab", + headers={"Content-Range": "bytes 0-1/notanumber"}, + ) + session = _make_session_router( + [ + ("HEAD", candidate.url, head_response), + ("GET", candidate.url, probe_response), + ], + ) + + with pytest.raises(MetadataFetchError, match="could not parse Content-Range"): + fetch_metadata(candidate, session) + + def test_windows_separator_metadata_member_is_found(self, tmp_path): + """A wheel that uses ``\\`` in entry names still resolves METADATA.""" + from pipenv.resolver.pure_python_metadata import fetch_metadata + + # zipfile uses the filename verbatim; we can write an entry with + # a backslash and verify the fallback ``endswith`` clause hits. + wheel_path = tmp_path / "winpkg-1.0.0-py3-none-any.whl" + metadata_text = ( + "Metadata-Version: 2.1\n" + "Name: winpkg\n" + "Version: 1.0.0\n" + "Requires-Dist: a\n" + "\n" + ) + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr( + "winpkg-1.0.0.dist-info\\METADATA", + metadata_text, + ) + zf.writestr("winpkg/__init__.py", "") + wheel_bytes = wheel_path.read_bytes() + + candidate = _make_wheel_candidate( + name="winpkg", + version="1.0.0", + url=( + "https://example.org/wheels/" + "winpkg-1.0.0-py3-none-any.whl" + ), + ) + + head_response = _make_response( + status=200, + headers={"Content-Length": str(len(wheel_bytes))}, + ) + get_response = _make_response( + status=206, + data=wheel_bytes, + headers={ + "Content-Range": ( + f"bytes 0-{len(wheel_bytes) - 1}/{len(wheel_bytes)}" + ), + }, + ) + session = _make_session_router( + [ + ("HEAD", candidate.url, head_response), + ("GET", candidate.url, get_response), + ], + ) + + result = fetch_metadata(candidate, session) + assert result.name == "winpkg" + assert "a" in result.requires_dist + + +# --------------------------------------------------------------------------- +# MetadataCache — additional coverage of failure / corruption modes +# --------------------------------------------------------------------------- + + +class TestMetadataCacheExtra: + """Corrupt-cache + write-failure paths.""" + + def _meta(self): + from pipenv.resolver.pure_python_metadata import CoreMetadata + + return CoreMetadata( + name="six", + version="1.16.0", + requires_python=">=2.7", + requires_dist=("attrs>=21.0",), + provides_extras=frozenset({"test"}), + summary="Compat utilities", + ) + + def test_cache_get_returns_none_for_malformed_json(self, tmp_path): + from pipenv.resolver.pure_python_metadata import MetadataCache + + cache = MetadataCache(tmp_path / "metadata-cache") + url = "https://example.org/wheels/foo.whl" + target = cache._path_for(url) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(b"{not really json") + assert cache.get(url) is None + + def test_cache_get_returns_none_for_non_dict_payload(self, tmp_path): + from pipenv.resolver.pure_python_metadata import MetadataCache + + cache = MetadataCache(tmp_path / "metadata-cache") + url = "https://example.org/wheels/foo.whl" + target = cache._path_for(url) + target.parent.mkdir(parents=True, exist_ok=True) + # Valid JSON but not a dict. + target.write_bytes(b'["not", "a", "dict"]') + assert cache.get(url) is None + + def test_cache_get_returns_none_for_wrong_schema_version(self, tmp_path): + import json as _json + + from pipenv.resolver.pure_python_metadata import MetadataCache + + cache = MetadataCache(tmp_path / "metadata-cache") + url = "https://example.org/wheels/foo.whl" + target = cache._path_for(url) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes( + _json.dumps( + { + "schema_version": 9999, + "name": "foo", + "version": "1.0", + } + ).encode("utf-8") + ) + assert cache.get(url) is None + + def test_cache_get_returns_none_for_missing_required_key(self, tmp_path): + """``schema_version=1`` but no ``name`` → cache treats as miss.""" + import json as _json + + from pipenv.resolver.pure_python_metadata import MetadataCache + + cache = MetadataCache(tmp_path / "metadata-cache") + url = "https://example.org/wheels/foo.whl" + target = cache._path_for(url) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes( + _json.dumps( + { + "schema_version": 1, + # no "name" + "version": "1.0", + } + ).encode("utf-8") + ) + assert cache.get(url) is None + + def test_cache_get_returns_none_on_oserror(self, tmp_path, monkeypatch): + """Generic ``OSError`` on read (not ``FileNotFoundError``) → miss.""" + from pathlib import Path as _Path + + from pipenv.resolver.pure_python_metadata import MetadataCache + + cache = MetadataCache(tmp_path / "metadata-cache") + + def _boom(self): # noqa: ARG001 + raise PermissionError("nope") + + monkeypatch.setattr(_Path, "read_bytes", _boom) + assert cache.get("https://example.org/wheels/foo.whl") is None + + def test_cache_get_returns_none_when_decode_fails(self, tmp_path): + """``UnicodeDecodeError`` inside ``json.loads`` → miss.""" + from pipenv.resolver.pure_python_metadata import MetadataCache + + cache = MetadataCache(tmp_path / "metadata-cache") + url = "https://example.org/wheels/foo.whl" + target = cache._path_for(url) + target.parent.mkdir(parents=True, exist_ok=True) + # Invalid UTF-8 → json.loads raises UnicodeDecodeError. + target.write_bytes(b"\xff\xfe\xfd") + assert cache.get(url) is None + + def test_cache_put_oserror_is_non_fatal(self, tmp_path, monkeypatch, caplog): + """A failing ``cache.put`` must not poison the returned metadata.""" + import logging as _logging + + from pipenv.resolver.pure_python_metadata import ( + MetadataCache, + fetch_metadata, + ) + + body = NUMPY_METADATA_TEXT.encode("utf-8") + body_hash = hashlib.sha256(body).hexdigest() + candidate = _make_wheel_candidate() + metadata_url = candidate.url + ".metadata" + + response = _make_response(status=200, data=body) + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + cache = MetadataCache(tmp_path / "metadata-cache") + + def _boom(self, wheel_url, metadata): # noqa: ARG001 + raise OSError("disk full") + + monkeypatch.setattr(MetadataCache, "put", _boom) + + caplog.set_level(_logging.DEBUG, logger="pipenv.resolver.pure_python_metadata") + result = fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + metadata_hash={"sha256": body_hash}, + cache=cache, + ) + # Metadata still returned despite write failure. + assert result.name == "numpy" + # Debug log emitted. + assert any( + "metadata cache write failed" in rec.message + for rec in caplog.records + ) + + def test_cache_put_cleans_tempfile_on_replace_failure( + self, tmp_path, monkeypatch + ): + """If ``os.replace`` raises, the temp file is unlinked.""" + import os as _os + + from pipenv.resolver.pure_python_metadata import MetadataCache + + cache = MetadataCache(tmp_path / "metadata-cache") + url = "https://example.org/wheels/six-1.16.0-py3-none-any.whl" + + real_replace = _os.replace + + def _boom_replace(src, dst): + raise OSError("simulated replace failure") + + monkeypatch.setattr(_os, "replace", _boom_replace) + + with pytest.raises(OSError, match="simulated replace failure"): + cache.put(url, self._meta()) + + # No temp file should be left behind: parent directory must be + # empty (the would-be target was never replaced and tmp was unlinked). + parent = cache._path_for(url).parent + leftovers = list(parent.iterdir()) + assert leftovers == [], f"unexpected leftover files: {leftovers}" + + # Restore for any later test in the class. + monkeypatch.setattr(_os, "replace", real_replace) + + def test_cache_key_is_url_dependent(self, tmp_path): + """Distinct URLs map to distinct paths; same URL → same path.""" + from pipenv.resolver.pure_python_metadata import MetadataCache + + cache = MetadataCache(tmp_path / "metadata-cache") + a = "https://example.org/wheels/a-1.0-py3-none-any.whl" + b = "https://example.org/wheels/b-1.0-py3-none-any.whl" + + path_a1 = cache._path_for(a) + path_a2 = cache._path_for(a) + path_b = cache._path_for(b) + + assert path_a1 == path_a2 + assert path_a1 != path_b + # SHA256 hex length is 64 + ``.json``. + assert path_a1.name.endswith(".json") + assert len(path_a1.stem) == 64 + + def test_cache_round_trip_via_disk_serialisation(self, tmp_path): + """``put`` + fresh ``MetadataCache`` reads it back identically.""" + from pipenv.resolver.pure_python_metadata import MetadataCache + + meta = self._meta() + url = "https://example.org/wheels/six.whl" + + writer = MetadataCache(tmp_path / "metadata-cache") + writer.put(url, meta) + + reader = MetadataCache(tmp_path / "metadata-cache") + out = reader.get(url) + assert out == meta + + +# --------------------------------------------------------------------------- +# HTTP helper coverage — _get_header in particular +# --------------------------------------------------------------------------- + + +class TestHttpHelpers: + def test_get_header_returns_none_for_none_headers(self): + from pipenv.resolver.pure_python_metadata import _get_header + + assert _get_header(None, "Content-Length") is None + + def test_get_header_direct_hit(self): + from pipenv.resolver.pure_python_metadata import _get_header + + assert _get_header({"Content-Length": "42"}, "Content-Length") == "42" + + def test_get_header_case_insensitive_fallback(self): + from pipenv.resolver.pure_python_metadata import _get_header + + # dict.get is case-sensitive — value lives behind the + # case-insensitive iteration fallback. Use a dict that *only* + # has the lowercase form, so the direct ``get`` returns None + # and the fallback kicks in. + class _PickyDict: + def __init__(self, data): + self._data = dict(data) + + def get(self, name): + # Strict case-sensitive lookup. + return self._data.get(name) + + def items(self): + return self._data.items() + + headers = _PickyDict({"content-length": "99"}) + assert _get_header(headers, "Content-Length") == "99" + + def test_get_header_items_raises_returns_none(self): + from pipenv.resolver.pure_python_metadata import _get_header + + class _Bad: + def get(self, name): # noqa: ARG002 + return None + + def items(self): + raise RuntimeError("not iterable") + + assert _get_header(_Bad(), "X") is None + + def test_get_header_get_raises_falls_through_to_items(self): + from pipenv.resolver.pure_python_metadata import _get_header + + class _Wonky: + def get(self, name): + raise RuntimeError("get unsupported") + + def items(self): + return [("Content-Length", "7")] + + assert _get_header(_Wonky(), "Content-Length") == "7" + + def test_get_header_no_match_returns_none(self): + from pipenv.resolver.pure_python_metadata import _get_header + + assert _get_header({"A": "1"}, "Z") is None + + def test_http_request_returns_none_on_exception(self): + from pipenv.resolver.pure_python_metadata import _http_request + + session = MagicMock() + session.request.side_effect = RuntimeError("boom") + assert _http_request(session, "GET", "https://x") is None + + def test_http_get_range_session_raises(self): + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + _http_get_range, + ) + + session = MagicMock() + session.request.side_effect = RuntimeError("net down") + with pytest.raises(MetadataFetchError, match="no response"): + _http_get_range(session, "https://x/y.whl", 0, 10) + + def test_http_get_range_non_206_or_200(self): + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + _http_get_range, + ) + + response = _make_response(status=416) + session = _make_session_router([("GET", "x/y", response)]) + with pytest.raises(MetadataFetchError, match="HTTP 416"): + _http_get_range(session, "https://x/y", 0, 10) + + def test_http_get_range_no_body(self): + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + _http_get_range, + ) + + response = _make_response(status=206) + response.data = None + session = _make_session_router([("GET", "x/y", response)]) + with pytest.raises(MetadataFetchError, match="no body"): + _http_get_range(session, "https://x/y", 0, 10) + + +# --------------------------------------------------------------------------- +# _PartialFile — the wheel-tail file-like adapter +# --------------------------------------------------------------------------- + + +class TestPartialFile: + def _build(self, *, tail=b"abcdef", offset=10, total=16, session=None): + from pipenv.resolver.pure_python_metadata import _PartialFile + + return _PartialFile( + tail, + offset=offset, + total_length=total, + session=session if session is not None else MagicMock(), + url="https://example.org/x.whl", + ) + + def test_seekable_readable_writable(self): + pf = self._build() + assert pf.seekable() is True + assert pf.readable() is True + assert pf.writable() is False + + def test_seek_set_cur_end_and_tell(self): + import io as _io + + pf = self._build() + assert pf.seek(5, _io.SEEK_SET) == 5 + assert pf.tell() == 5 + assert pf.seek(2, _io.SEEK_CUR) == 7 + assert pf.seek(-1, _io.SEEK_END) == 15 + with pytest.raises(ValueError, match="unsupported whence"): + pf.seek(0, 99) + with pytest.raises(ValueError, match="negative seek"): + pf.seek(-100, _io.SEEK_SET) + + def test_read_within_buffer(self): + """Read entirely within the tail buffer — no network.""" + session = MagicMock() + pf = self._build(tail=b"abcdef", offset=10, total=16, session=session) + pf.seek(10) + assert pf.read(3) == b"abc" + # No range fetches. + session.request.assert_not_called() + + def test_read_negative_size_to_eof(self): + session = MagicMock() + pf = self._build(tail=b"abcdef", offset=10, total=16, session=session) + pf.seek(13) + assert pf.read(-1) == b"def" + + def test_read_none_size_to_eof(self): + session = MagicMock() + pf = self._build(tail=b"abcdef", offset=10, total=16, session=session) + pf.seek(13) + assert pf.read(None) == b"def" # type: ignore[arg-type] + + def test_read_at_or_past_end_returns_empty(self): + pf = self._build() + pf.seek(16) # at EOF + assert pf.read(10) == b"" + pf.seek(100) # past EOF + assert pf.read(10) == b"" + + def test_read_size_zero_returns_empty(self): + pf = self._build() + pf.seek(10) + assert pf.read(0) == b"" + + def test_read_prefix_fetch_grows_buffer_low(self): + """Reading before ``_buffer_start`` issues a prefix range GET.""" + from pipenv.resolver.pure_python_metadata import _PartialFile + + # Full wheel bytes laid out: positions 0..19 are "0123456789ABCDEFGHIJ". + full = b"0123456789ABCDEFGHIJ" + tail = full[10:] # buffer covers [10, 20) + + prefix_response = _make_response( + status=206, + data=full[0:10], + headers={"Content-Range": "bytes 0-9/20"}, + ) + session = _make_session_router( + [("GET", "example.org/x.whl", prefix_response)], + ) + + pf = _PartialFile( + tail, + offset=10, + total_length=20, + session=session, + url="https://example.org/x.whl", + ) + pf.seek(0) + out = pf.read(10) + assert out == b"0123456789" + # After the fetch the buffer covers [0, 20). + assert pf._buffer_start == 0 + assert bytes(pf._buffer) == full + + def test_read_prefix_oversized_response_trimmed(self): + """Mirror returns the full body on a range GET → we trim to ``wanted``.""" + from pipenv.resolver.pure_python_metadata import _PartialFile + + full = b"0123456789ABCDEFGHIJ" + tail = full[10:] + # Server ignores Range and returns the entire wheel. + prefix_response = _make_response(status=200, data=full) + session = _make_session_router( + [("GET", "example.org/x.whl", prefix_response)], + ) + + pf = _PartialFile( + tail, + offset=10, + total_length=20, + session=session, + url="https://example.org/x.whl", + ) + pf.seek(0) + out = pf.read(10) + assert out == b"0123456789" + + def test_read_prefix_short_read_raises(self): + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + _PartialFile, + ) + + tail = b"ABCDEFGHIJ" + # We will request 10 bytes for the prefix but the mirror only + # returns 3. + prefix_response = _make_response(status=206, data=b"012") + session = _make_session_router( + [("GET", "example.org/x.whl", prefix_response)], + ) + + pf = _PartialFile( + tail, + offset=10, + total_length=20, + session=session, + url="https://example.org/x.whl", + ) + pf.seek(0) + with pytest.raises(MetadataFetchError, match="short range read"): + pf.read(10) + + def test_read_suffix_fetch_grows_buffer_high(self): + from pipenv.resolver.pure_python_metadata import _PartialFile + + # Initial buffer covers [0, 10); request a read into [10, 20). + head = b"0123456789" + suffix_response = _make_response( + status=206, + data=b"ABCDEFGHIJ", + headers={"Content-Range": "bytes 10-19/20"}, + ) + session = _make_session_router( + [("GET", "example.org/x.whl", suffix_response)], + ) + + pf = _PartialFile( + head, + offset=0, + total_length=20, + session=session, + url="https://example.org/x.whl", + ) + pf.seek(10) + out = pf.read(10) + assert out == b"ABCDEFGHIJ" + + def test_read_suffix_oversized_response_trimmed(self): + from pipenv.resolver.pure_python_metadata import _PartialFile + + head = b"0123456789" + # Mirror returns way more than requested. + suffix_response = _make_response( + status=206, data=b"ABCDEFGHIJ_extra_junk_bytes" + ) + session = _make_session_router( + [("GET", "example.org/x.whl", suffix_response)], + ) + + pf = _PartialFile( + head, + offset=0, + total_length=20, + session=session, + url="https://example.org/x.whl", + ) + pf.seek(10) + out = pf.read(10) + assert out == b"ABCDEFGHIJ" + + def test_read_suffix_short_read_raises(self): + from pipenv.resolver.pure_python_metadata import ( + MetadataFetchError, + _PartialFile, + ) + + head = b"0123456789" + suffix_response = _make_response(status=206, data=b"AB") + session = _make_session_router( + [("GET", "example.org/x.whl", suffix_response)], + ) + + pf = _PartialFile( + head, + offset=0, + total_length=20, + session=session, + url="https://example.org/x.whl", + ) + pf.seek(10) + with pytest.raises(MetadataFetchError, match="short range read"): + pf.read(10) + + +# --------------------------------------------------------------------------- +# T_S2 — sdist routing inside fetch_metadata +# --------------------------------------------------------------------------- + + +def _make_sdist_candidate( + name: str = "examplepkg", + version: str = "1.0.0", + *, + url: str = "https://example.org/sdists/examplepkg-1.0.0.tar.gz", +) -> Candidate: + """Build a sdist :class:`Candidate` (``is_wheel == False``). + + Mirrors :func:`_make_wheel_candidate` but with a ``.tar.gz`` + filename, so :meth:`Candidate.from_filename` derives + ``is_wheel = False`` and ``wheel_tags = None``. + """ + filename = url.rsplit("/", 1)[-1] + return Candidate.from_filename( + filename, + name=name, + version=version, + url=url, + hashes=frozenset(), + requires_python=">=3.9", + yanked=False, + yanked_reason=None, + upload_time=None, + ) + + +class TestFetchMetadataSdistRouting: + """T_S2 — ``fetch_metadata`` branches on ``candidate.is_wheel``. + + Wheel candidates keep the existing PEP 658 + wheel-head fallback + paths; sdist candidates delegate to T_S1's + :func:`pipenv.resolver.pure_python_sdist.extract_metadata_from_sdist`. + Cache is shared — same :class:`MetadataCache` keyed by URL sha256. + """ + + def test_sdist_candidate_routes_to_sdist_extractor(self): + """``is_wheel=False`` candidate → extract_metadata_from_sdist invoked. + + No HTTP HEAD/GET is performed on the sdist URL: the wheel-head + fallback machinery must not fire on a sdist candidate. The + session router has zero routes, so any attempted HTTP call + would raise ``AssertionError`` — passing this test confirms the + wheel path is bypassed entirely. + """ + from unittest.mock import patch + + from pipenv.resolver.pure_python_metadata import ( + CoreMetadata, + fetch_metadata, + ) + + candidate = _make_sdist_candidate() + assert candidate.is_wheel is False # sanity-check the helper + + session = _make_session_router([]) # no HTTP expected at all + + expected_metadata = CoreMetadata( + name="examplepkg", + version="1.0.0", + requires_python=None, + requires_dist=(), + provides_extras=frozenset(), + summary=None, + ) + + with patch( + "pipenv.resolver.pure_python_sdist.extract_metadata_from_sdist", + return_value=expected_metadata, + ) as mock_extract: + result = fetch_metadata(candidate, session) + + mock_extract.assert_called_once() + # First positional arg is the candidate; second is the session. + call_args, call_kwargs = mock_extract.call_args + assert call_args[0] is candidate + assert call_args[1] is session + # cache kwarg is forwarded (None when not supplied by caller). + assert call_kwargs.get("cache") is None + # Result is propagated verbatim — same dataclass instance. + assert result is expected_metadata + # Hard guarantee: no HTTP traffic on the sdist URL. + assert session.request.call_count == 0 + + def test_wheel_candidate_does_not_route_to_sdist(self): + """``is_wheel=True`` ⇒ wheel path; sdist extractor never invoked. + + Pin via a ``patch`` on ``extract_metadata_from_sdist`` whose + ``side_effect`` raises if called — combined with a regular + PEP 658 happy path, this proves the wheel branch is preserved. + """ + from unittest.mock import patch + + from pipenv.resolver.pure_python_metadata import fetch_metadata + + body = NUMPY_METADATA_TEXT.encode("utf-8") + body_hash = hashlib.sha256(body).hexdigest() + + candidate = _make_wheel_candidate() + assert candidate.is_wheel is True # sanity-check the helper + + metadata_url = candidate.url + ".metadata" + response = _make_response(status=200, data=body) + session = _make_session_router( + [("GET", metadata_url, response)], + ) + + def _explode(*_args, **_kwargs): + raise AssertionError( + "extract_metadata_from_sdist must not be called for a wheel" + ) + + with patch( + "pipenv.resolver.pure_python_sdist.extract_metadata_from_sdist", + side_effect=_explode, + ) as mock_extract: + result = fetch_metadata( + candidate, + session, + metadata_url=metadata_url, + metadata_hash={"sha256": body_hash}, + ) + + mock_extract.assert_not_called() + # Result came from the wheel path — parsed real METADATA. + assert result.name == "numpy" + assert result.version == "1.26.0" + + def test_sdist_cache_passed_through(self, tmp_path): + """``cache`` kwarg is forwarded to the sdist extractor. + + T_S1 honours / populates the shared :class:`MetadataCache` + internally; T_S2 only needs to pass it through so the on-disk + cache key (``sha256(candidate.url)``) is shared across wheel + and sdist routes. Also covers the cache short-circuit: a + pre-populated cache entry must skip both the sdist extractor + and any HTTP traffic. + """ + from unittest.mock import patch + + from pipenv.resolver.pure_python_metadata import ( + CoreMetadata, + MetadataCache, + fetch_metadata, + ) + + candidate = _make_sdist_candidate() + session = _make_session_router([]) + + cache = MetadataCache(tmp_path / "metadata-cache") + + # --- first call: miss → extractor invoked → cache kwarg forwarded. + expected_metadata = CoreMetadata( + name="examplepkg", + version="1.0.0", + requires_python=None, + requires_dist=(), + provides_extras=frozenset(), + summary=None, + ) + + with patch( + "pipenv.resolver.pure_python_sdist.extract_metadata_from_sdist", + return_value=expected_metadata, + ) as mock_extract: + result = fetch_metadata(candidate, session, cache=cache) + + mock_extract.assert_called_once() + _, call_kwargs = mock_extract.call_args + assert call_kwargs.get("cache") is cache + assert result is expected_metadata + + # --- second call: pre-populated cache → extractor never invoked. + # T_S1 owns its own cache.put; emulate that here so we can + # exercise fetch_metadata's own cache short-circuit branch + # without depending on the real extractor's I/O. + cache.put(candidate.url, expected_metadata) + + with patch( + "pipenv.resolver.pure_python_sdist.extract_metadata_from_sdist", + side_effect=AssertionError( + "extractor must not be called on a cache hit" + ), + ) as mock_extract2: + cached_result = fetch_metadata(candidate, session, cache=cache) + + mock_extract2.assert_not_called() + assert cached_result == expected_metadata + # Still no HTTP traffic. + assert session.request.call_count == 0 diff --git a/tests/unit/test_pure_python_provider.py b/tests/unit/test_pure_python_provider.py new file mode 100644 index 0000000000..5a9a2c8e92 --- /dev/null +++ b/tests/unit/test_pure_python_provider.py @@ -0,0 +1,2509 @@ +"""Focused RED→GREEN tests for :class:`PurePythonProvider` methods +(Initiative G phase 3, T3 + T4 + T5 + T6). + +Scope: T3 (``identify``) + T4 (``find_matches``) + T5 +(``get_preference``) + T6 (``is_satisfied_by``). T13 will extend this +file with full per-method coverage of ``get_dependencies`` (T7 +deliverable). + +T3 bullets: + +1. ``identify(Requirement(name="django", extras=frozenset({"argon2"}), ...))`` + returns ``("django", frozenset({"argon2"}))``. +2. ``identify(Candidate(name="django", ...))`` returns + ``("django", frozenset())``. +3. Round-trip equality — two requirements with the same name + extras + produce equal ``identify`` outputs (this is the ``resolvelib`` + contract: same identifier = same identity bucket). + +T4 bullets (per ``initiative-g-phase3-plan.md`` T4 validation matrix): + +1. Mock cache returns 5 :class:`Candidate`\\ s for ``django``; + ``find_matches`` with specifier ``>=4.0`` returns only those with + version ≥ 4.0, highest first. +2. Mock cache returns 0 candidates; ``find_matches`` returns an empty + iterable without raising. +3. ``incompatibilities`` filter: pass one of the returned candidates as + incompatible; it is excluded from the output. +4. ``requires_python`` filter: a candidate whose ``requires_python`` is + incompatible with ``target_env["python_version"]`` is excluded. + +All ``packaging`` imports come from :mod:`pipenv.vendor.packaging` +(vendored, not patched pip). Zero ``pip._internal.*`` imports are +permitted in this test file or the module it exercises — the Phase 1 +pre-commit gate enforces the constraint on the module; this file +mirrors the discipline for consistency. +""" +from __future__ import annotations + +from typing import Any + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_requirement( + *, + name: str = "django", + spec: str = "", + extras: frozenset[str] = frozenset(), + source: str = "pipfile", + parent: str | None = None, +): + """Build a :class:`Requirement` with one knob per axis the test + cares about. Keeps the test bodies focused on the identifier + contract rather than on dataclass-construction ceremony. + """ + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.specifiers import SpecifierSet + + return Requirement( + name=name, + specifier=SpecifierSet(spec), + extras=extras, + marker=None, + source=source, # type: ignore[arg-type] + parent=parent, + ) + + +def _make_candidate( + *, + name: str = "django", + version: str = "4.2.0", + extras: frozenset[str] | None = None, +): + """Build a :class:`Candidate` with the minimum-viable fields the + provider's ``identify`` reads. ``extras`` is forwarded only when + the caller supplies a non-``None`` value so we exercise the + default-factory path (no kwarg) AND the explicit-kwarg path.""" + from pipenv.resolver.candidate import Candidate + + kwargs: dict[str, Any] = { + "name": name, + "version": version, + "url": f"https://example.org/{name}-{version}-py3-none-any.whl", + "filename": f"{name}-{version}-py3-none-any.whl", + "hashes": frozenset(), + "requires_python": None, + "yanked": False, + "yanked_reason": None, + "upload_time": None, + "is_wheel": True, + "wheel_tags": None, + } + if extras is not None: + kwargs["extras"] = extras + return Candidate(**kwargs) + + +def _make_provider( + *, + cache: Any = None, + fetcher: Any = None, + metadata_fetcher: Any = None, + target_env: Any = None, + index_urls: list[str] | None = None, + allow_prereleases: bool = False, +): + """Construct a :class:`PurePythonProvider` with dummy dependencies. + + T3 doesn't touch the cache / fetcher / metadata_fetcher / target_env + fields (later tasks will), so simple sentinels suffice. T4 adds + ``index_urls`` + ``allow_prereleases`` keyword arguments — both + default to a one-element list (``["https://pypi.org/simple"]``) and + ``False`` so existing T3 tests construct without changes.""" + from pipenv.resolver.pure_python_provider import PurePythonProvider + + return PurePythonProvider( + cache=object() if cache is None else cache, + fetcher=object() if fetcher is None else fetcher, + metadata_fetcher=( + object() if metadata_fetcher is None else metadata_fetcher + ), + target_env={} if target_env is None else target_env, + index_urls=( + ["https://pypi.org/simple"] if index_urls is None else index_urls + ), + allow_prereleases=allow_prereleases, + ) + + +# --------------------------------------------------------------------------- +# T4 helpers — fake cache + fetcher +# --------------------------------------------------------------------------- + + +class _FakeCache: + """In-memory stand-in for :class:`ParsedManifestCache`. + + Behaves like the real ``cache.get(index_url, package_name)``: returns + ``None`` on miss, a tiny stand-in for :class:`CachedManifest` + (anything exposing ``.candidates``) on hit. + + Tests seed ``_data`` directly with ``{(index_url, name): tuple(cands)}``. + """ + + class _Manifest: + __slots__ = ("candidates",) + + def __init__(self, candidates): + self.candidates = tuple(candidates) + + def __init__(self, data=None): + self._data: dict[tuple[str, str], tuple] = dict(data or {}) + self.get_calls: list[tuple[str, str]] = [] + + def get(self, index_url: str, package_name: str): + self.get_calls.append((index_url, package_name)) + cands = self._data.get((index_url, package_name)) + if cands is None: + return None + return self._Manifest(cands) + + def seed(self, index_url: str, package_name: str, candidates) -> None: + self._data[(index_url, package_name)] = tuple(candidates) + + +class _FakeFetcher: + """In-memory stand-in for :class:`ParallelFetcher`. + + ``populate(targets)`` records the targets it was asked to fetch and + seeds the linked cache from a pre-supplied ``_payload`` map of + ``{(index_url, name): tuple(cands)}``. T4's find_matches will call + ``self._fetcher.populate(...)`` on cache miss; the test asserts on + ``populate_calls`` and on the post-call cache contents. + """ + + def __init__(self, cache: _FakeCache, payload=None): + self._cache = cache + self._payload: dict[tuple[str, str], tuple] = dict(payload or {}) + self.populate_calls: list[list[tuple[str, str]]] = [] + + def populate(self, targets): + target_list = list(targets) + self.populate_calls.append(target_list) + for index_url, name in target_list: + cands = self._payload.get((index_url, name)) + if cands is not None: + self._cache.seed(index_url, name, cands) + return {} + + +def _cand( + *, + name: str = "django", + version: str = "4.2.0", + requires_python: str | None = None, + is_wheel: bool = True, + extras: frozenset[str] = frozenset(), + yanked: bool = False, + yanked_reason: str | None = None, +): + """Build a :class:`Candidate` for T4 tests with the four fields + ``find_matches`` actually reads (``name``, ``version``, + ``requires_python``, ``extras``) parameterised and the rest pinned + to inert defaults.""" + from pipenv.resolver.candidate import Candidate + + suffix = "py3-none-any.whl" if is_wheel else "tar.gz" + filename = f"{name}-{version}-{suffix}" if is_wheel else f"{name}-{version}.tar.gz" + return Candidate( + name=name, + version=version, + url=f"https://example.org/{filename}", + filename=filename, + hashes=frozenset(), + requires_python=requires_python, + yanked=yanked, + yanked_reason=yanked_reason, + upload_time=None, + is_wheel=is_wheel, + wheel_tags=None, + extras=extras, + ) + + +# --------------------------------------------------------------------------- +# T3 validation matrix — one test per bullet +# --------------------------------------------------------------------------- + + +class TestIdentifyRequirement: + """Bullet 1: ``identify`` on a :class:`Requirement` returns + ``(canonical_name, frozenset(extras))``.""" + + def test_requirement_with_extras_returns_name_and_extras(self): + provider = _make_provider() + req = _make_requirement( + name="django", extras=frozenset({"argon2"}) + ) + assert provider.identify(req) == ("django", frozenset({"argon2"})) + + def test_requirement_without_extras_returns_empty_frozenset(self): + provider = _make_provider() + req = _make_requirement(name="requests", extras=frozenset()) + assert provider.identify(req) == ("requests", frozenset()) + + def test_requirement_with_multiple_extras_returns_all(self): + provider = _make_provider() + req = _make_requirement( + name="django", extras=frozenset({"argon2", "bcrypt"}) + ) + assert provider.identify(req) == ( + "django", + frozenset({"argon2", "bcrypt"}), + ) + + +class TestIdentifyCandidate: + """Bullet 2: ``identify`` on a :class:`Candidate` returns + ``(name, frozenset())`` when the candidate carries no extras. + + The Phase 1 ``Candidate`` shape didn't have an ``extras`` field; + T3 widens it with a default-factory ``frozenset()`` so existing + construction sites stay non-breaking. This test pins both paths: + the default-factory case (no kwarg) and the explicit-empty case.""" + + def test_candidate_default_extras_returns_empty_frozenset(self): + provider = _make_provider() + cand = _make_candidate(name="django") + assert provider.identify(cand) == ("django", frozenset()) + + def test_candidate_explicit_empty_extras_returns_empty_frozenset(self): + provider = _make_provider() + cand = _make_candidate(name="django", extras=frozenset()) + assert provider.identify(cand) == ("django", frozenset()) + + +class TestIdentifyRoundTripEquality: + """Bullet 3: two requirements with the same name + extras have + equal ``identify`` outputs. This is the load-bearing ``resolvelib`` + invariant — same identifier = same identity bucket in the dep graph. + + We also pin the symmetrical claim that a :class:`Requirement` and a + :class:`Candidate` with the same ``(name, extras)`` produce equal + identifiers — that's what lets ``resolvelib`` match candidates back + to requirements via the shared key. + """ + + def test_two_requirements_same_name_same_extras_equal_identify(self): + provider = _make_provider() + req1 = _make_requirement( + name="django", extras=frozenset({"argon2"}) + ) + req2 = _make_requirement( + name="django", + spec=">=4.0", # different specifier — identifier should be + # spec-agnostic + extras=frozenset({"argon2"}), + source="transitive", # different source — identifier should + # also be source-agnostic + ) + assert provider.identify(req1) == provider.identify(req2) + + def test_two_requirements_same_name_different_extras_distinct(self): + """Different extras → different identifier. Mirrors pip's + grouping where ``django`` and ``django[argon2]`` are distinct + graph nodes.""" + provider = _make_provider() + req1 = _make_requirement(name="django", extras=frozenset()) + req2 = _make_requirement( + name="django", extras=frozenset({"argon2"}) + ) + assert provider.identify(req1) != provider.identify(req2) + + def test_requirement_and_candidate_same_name_extras_equal_identify(self): + """Cross-shape equality: a requirement and a candidate sharing + the ``(name, extras)`` pair produce the same identifier. This + is what lets ``resolvelib`` route a candidate back to the + requirement bucket that asked for it.""" + provider = _make_provider() + req = _make_requirement(name="django", extras=frozenset()) + cand = _make_candidate(name="django") + assert provider.identify(req) == provider.identify(cand) + + +# --------------------------------------------------------------------------- +# T4 validation matrix — one test per bullet +# --------------------------------------------------------------------------- + + +_INDEX = "https://pypi.org/simple" + + +class TestFindMatchesSpecifierFilter: + """Bullet 1: with 5 cached candidates and a ``>=4.0`` specifier, + only the candidates with version ≥ 4.0 are returned, ordered + highest-version-first.""" + + def test_specifier_filter_returns_high_version_first(self): + cands = [ + _cand(name="django", version="3.2.0"), + _cand(name="django", version="4.0.0"), + _cand(name="django", version="4.1.0"), + _cand(name="django", version="4.2.0"), + _cand(name="django", version="5.0.0"), + ] + cache = _FakeCache({(_INDEX, "django"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + + versions = [c.version for c in result] + # 3.2.0 is filtered; remaining four returned high-version-first. + assert versions == ["5.0.0", "4.2.0", "4.1.0", "4.0.0"] + + +class TestFindMatchesEmpty: + """Bullet 2: with an empty cache (zero candidates), ``find_matches`` + returns an empty iterable and does not raise. + + The cache lookup miss path triggers ``fetcher.populate`` — which we + stub to also return zero candidates so the second cache read also + misses. No exception.""" + + def test_empty_cache_returns_empty_without_raise(self): + cache = _FakeCache() + fetcher = _FakeFetcher(cache, payload={}) + provider = _make_provider( + cache=cache, fetcher=fetcher, index_urls=[_INDEX] + ) + + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + assert result == [] + # ``populate`` was invoked because the cache was cold. + assert fetcher.populate_calls == [[(_INDEX, "django")]] + + +class TestFindMatchesIncompatibilitiesFilter: + """Bullet 3: a candidate listed in ``incompatibilities`` is excluded + from the result. + + Match keys on ``(name, version, filename)`` so the incompatible + candidate doesn't need to be the same Python object as the cached + one — same identity tuple is enough. + """ + + def test_incompatible_candidate_excluded(self): + cands = [ + _cand(name="django", version="4.0.0"), + _cand(name="django", version="4.1.0"), + _cand(name="django", version="4.2.0"), + ] + cache = _FakeCache({(_INDEX, "django"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + # Use the same candidate object so dedup keys line up. + incompat = cands[1] # 4.1.0 + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={identifier: iter([incompat])}, + ) + ) + versions = [c.version for c in result] + assert versions == ["4.2.0", "4.0.0"] + + +class TestFindMatchesRequiresPythonFilter: + """Bonus bullet: a candidate whose ``requires_python`` excludes the + target ``python_version`` is filtered out. + + Target env is ``python_version="3.12"``; candidate advertises + ``requires_python=">=4.0"`` so it should be dropped. A second + candidate with no ``requires_python`` advertisement remains. + """ + + def test_requires_python_incompatible_excluded(self): + cands = [ + _cand(name="django", version="5.0.0", requires_python=">=4.0"), + _cand(name="django", version="4.2.0", requires_python=None), + _cand(name="django", version="4.0.0", requires_python=">=3.10"), + ] + cache = _FakeCache({(_INDEX, "django"): tuple(cands)}) + provider = _make_provider( + cache=cache, + index_urls=[_INDEX], + target_env={"python_version": "3.12"}, + ) + + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + versions = [c.version for c in result] + # 5.0.0 is excluded (requires_python=">=4.0" rejects "3.12"). + # 4.2.0 (no requires_python) and 4.0.0 (>=3.10 accepts 3.12) remain. + assert versions == ["4.2.0", "4.0.0"] + + +class TestFindMatchesPrefersWheelOverSdistAtSameVersion: + """When a release has both a wheel and an sdist, ``find_matches`` + must put the wheel first so the resolver prefers it — mirrors pip's + ``PackageFinder`` ordering. + + Regression test for the bench-fixture failures where + ``python3-saml 1.16.0``, ``redis-py-cluster 2.1.3``, and friends + all advertise both a wheel AND an sdist at the picked version, but + pure-python's version-only sort key routed to the sdist artifact — + forcing a PEP 517 build for nothing (and crashing on packages + whose sdist build backend isn't trivially installable). + """ + + def test_wheel_sorts_before_sdist_at_same_version(self): + # Wheel-priority sort + per-version sdist suppression: when + # both a wheel and an sdist exist at the same release, only + # the wheel is returned (the sdist is dropped — see + # ``test_sdist_dropped_when_wheel_exists_at_same_version``). + cands = [ + _cand(name="python3-saml", version="1.16.0", is_wheel=False), + _cand(name="python3-saml", version="1.16.0", is_wheel=True), + ] + cache = _FakeCache({(_INDEX, "python3-saml"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + identifier = ("python3-saml", frozenset()) + req = _make_requirement(name="python3-saml", spec=">=1.15") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + assert [c.is_wheel for c in result] == [True] + + def test_higher_version_wins_over_artifact_kind(self): + # 1.17.0 sdist still beats 1.16.0 wheel — version is the + # primary sort key; artifact preference is the tiebreaker. + # 1.16.0 sdist is suppressed (per-version sdist drop) because + # 1.16.0 also has a wheel; 1.17.0 sdist survives (no wheel + # for that release). + cands = [ + _cand(name="pkg", version="1.16.0", is_wheel=True), + _cand(name="pkg", version="1.17.0", is_wheel=False), + ] + cache = _FakeCache({(_INDEX, "pkg"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + identifier = ("pkg", frozenset()) + req = _make_requirement(name="pkg", spec=">=1.15") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + assert [(c.version, c.is_wheel) for c in result] == [ + ("1.17.0", False), + ("1.16.0", True), + ] + + def test_sdist_dropped_when_wheel_exists_at_same_version(self): + # Per-version sdist suppression: returning both a wheel and an + # sdist at the same release lets ``resolvelib`` try the sdist + # as a backtrack candidate after a downstream conflict on the + # wheel — same metadata, same conflict, wasted PEP 517 builds. + # Mirrors pip's ``PackageFinder`` (one canonical Link per + # version). Bench-fixture trigger: ``sentry-protos`` / + # ``grpcio-status`` exploding into dozens of sdist builds as + # the resolver backtracked. + cands = [ + _cand(name="sentry-protos", version="0.8.32", is_wheel=True), + _cand(name="sentry-protos", version="0.8.32", is_wheel=False), + _cand(name="sentry-protos", version="0.8.31", is_wheel=True), + _cand(name="sentry-protos", version="0.8.31", is_wheel=False), + _cand(name="sentry-protos", version="0.8.30", is_wheel=False), + ] + cache = _FakeCache({(_INDEX, "sentry-protos"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + identifier = ("sentry-protos", frozenset()) + req = _make_requirement(name="sentry-protos", spec=">=0.8") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + # 0.8.32 → wheel (sdist dropped); 0.8.31 → wheel (sdist + # dropped); 0.8.30 → sdist (no wheel exists for it). + assert [(c.version, c.is_wheel) for c in result] == [ + ("0.8.32", True), + ("0.8.31", True), + ("0.8.30", False), + ] + + +class TestFindMatchesYankedFilter: + """PEP 592: yanked candidates are filtered from automatic selection + unless the user explicitly pins to that exact version via ``==``. + + Regression test for the bench-fixture lock failure where + ``sentry-relay==1.1.4`` (every artifact yanked with reason + "accidental release") got picked as the highest-version match for + the spec ``sentry-relay>=0.8.45`` and crashed the sdist build. + Pip's resolver skips it; pure-python now does too. + """ + + def test_yanked_excluded_from_range_spec(self): + cands = [ + _cand(name="sentry-relay", version="1.1.4", yanked=True), + _cand(name="sentry-relay", version="0.9.27"), + _cand(name="sentry-relay", version="0.8.45"), + ] + cache = _FakeCache({(_INDEX, "sentry-relay"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + identifier = ("sentry-relay", frozenset()) + req = _make_requirement(name="sentry-relay", spec=">=0.8.45") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + versions = [c.version for c in result] + # 1.1.4 (yanked) must NOT appear; 0.9.27 sorts first, then 0.8.45. + assert versions == ["0.9.27", "0.8.45"] + + def test_yanked_kept_when_exact_pinned(self): + # An explicit ``==1.1.4`` pin opts in to the yanked release — + # PEP 592 §"the user explicitly references it by version". + cands = [ + _cand(name="sentry-relay", version="1.1.4", yanked=True), + _cand(name="sentry-relay", version="0.9.27"), + ] + cache = _FakeCache({(_INDEX, "sentry-relay"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + identifier = ("sentry-relay", frozenset()) + req = _make_requirement(name="sentry-relay", spec="==1.1.4") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + assert [c.version for c in result] == ["1.1.4"] + + def test_yanked_excluded_when_pinned_to_other_version(self): + # ``==0.9.27`` doesn't license the yanked 1.1.4 either — + # ``_candidate_satisfies_requirements`` rejects 1.1.4 already, + # but the yanked filter is the load-bearing one when the spec + # is permissive. + cands = [ + _cand(name="sentry-relay", version="1.1.4", yanked=True), + _cand(name="sentry-relay", version="0.9.27"), + ] + cache = _FakeCache({(_INDEX, "sentry-relay"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + identifier = ("sentry-relay", frozenset()) + req = _make_requirement(name="sentry-relay", spec="==0.9.27") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + assert [c.version for c in result] == ["0.9.27"] + + +# --------------------------------------------------------------------------- +# T5 — get_preference (Q-C strict mirror of pip's tuple shape) +# --------------------------------------------------------------------------- + + +def _ri(requirement, parent=None): + """Build a ``RequirementInformation`` namedtuple as ``resolvelib`` + hands one to ``get_preference``. + + The vendored ``resolvelib.structs.RequirementInformation`` is a + ``namedtuple("RequirementInformation", ["requirement", "parent"])`` + — we instantiate the vendored type so the tests exercise the same + shape ``resolvelib.Resolver`` produces at runtime. + """ + from pipenv.patched.pip._vendor.resolvelib.structs import ( + RequirementInformation, + ) + + return RequirementInformation(requirement=requirement, parent=parent) + + +class TestGetPreferencePinnedVsRange: + """Bullet 1 (per plan T5 validation matrix): a pinned requirement + (``==4.0.1``) sorts before a range requirement (``>=4.0``). + + Q-C strict mirror: in pip's tuple, ``not pinned`` is one of the + leading components — ``False < True``, so pinned (False) sorts + before non-pinned (True). Our mirror keeps the same ordering. + """ + + def test_pinned_sorts_before_range(self): + provider = _make_provider() + pinned_id = ("alpha", frozenset()) + range_id = ("beta", frozenset()) + pinned_req = _make_requirement(name="alpha", spec="==4.0.1") + range_req = _make_requirement(name="beta", spec=">=4.0") + information = { + pinned_id: iter([_ri(pinned_req)]), + range_id: iter([_ri(range_req)]), + } + pref_pinned = provider.get_preference( + pinned_id, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=(), + ) + # Rebuild information — iterators are one-shot. + information = { + pinned_id: iter([_ri(pinned_req)]), + range_id: iter([_ri(range_req)]), + } + pref_range = provider.get_preference( + range_id, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=(), + ) + assert pref_pinned < pref_range + + def test_wildcard_eq_is_not_pinned(self): + """``==4.*`` is a wildcard equality — pip treats it as + upper-bounded, not pinned. Mirror that: a wildcard-equality + requirement does NOT get the pinned-first preference.""" + provider = _make_provider() + wildcard_id = ("alpha", frozenset()) + pinned_id = ("beta", frozenset()) + wildcard_req = _make_requirement(name="alpha", spec="==4.*") + pinned_req = _make_requirement(name="beta", spec="==4.0.1") + information = { + wildcard_id: iter([_ri(wildcard_req)]), + pinned_id: iter([_ri(pinned_req)]), + } + pref_wildcard = provider.get_preference( + wildcard_id, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=(), + ) + information = { + wildcard_id: iter([_ri(wildcard_req)]), + pinned_id: iter([_ri(pinned_req)]), + } + pref_pinned = provider.get_preference( + pinned_id, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=(), + ) + # Pinned (==4.0.1, no wildcard) sorts before wildcard (==4.*). + assert pref_pinned < pref_wildcard + + +class TestGetPreferencePipfileVsTransitive: + """Bullet 2: a Pipfile-direct requirement sorts before a transitive + requirement on the same axis. + + Note on Q-C strict mirror: pip's "direct" component refers to + ``ExplicitRequirement`` (URL-based / direct-reference requirements), + NOT Pipfile-direct. Our typed :class:`Requirement` doesn't model + URL-direct yet (T1 only encodes ``source`` ∈ + ``{"pipfile", "transitive", "constraint"}``), so we mirror pip's + "direct" slot by treating ``source == "pipfile"`` as the + closest-available analog — this is the design-doc summary's + rendering of the same idea (see plan T5 + design §5.3). When T1 + gains a URL-direct shape (Phase 4 work), the parity-divergence + note in the T5 plan entry tracks the upgrade. + """ + + def test_pipfile_sorts_before_transitive(self): + provider = _make_provider() + pipfile_id = ("alpha", frozenset()) + transitive_id = ("beta", frozenset()) + pipfile_req = _make_requirement( + name="alpha", spec=">=1.0", source="pipfile" + ) + transitive_req = _make_requirement( + name="beta", + spec=">=1.0", + source="transitive", + parent="some-parent", + ) + information = { + pipfile_id: iter([_ri(pipfile_req)]), + transitive_id: iter([_ri(transitive_req)]), + } + pref_pipfile = provider.get_preference( + pipfile_id, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=(), + ) + information = { + pipfile_id: iter([_ri(pipfile_req)]), + transitive_id: iter([_ri(transitive_req)]), + } + pref_transitive = provider.get_preference( + transitive_id, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=(), + ) + assert pref_pipfile < pref_transitive + + +class TestGetPreferenceBacktrackCount: + """Bullet 3: an identifier appearing in ``backtrack_causes`` is + PROMOTED — sorts BEFORE an identifier that has not caused + backtracking. + + Mirrors pip's ``not conflict_promoted`` slot in + ``PipProvider.get_preference`` — pip puts the promoted-set signal + at the head of the tuple so a conflict-causing identifier surfaces + its conflict early, instead of pinning a wide unconflicted layer + only to roll it all back later. Our analog uses the + ``backtrack_causes`` arg as the "promoted" signal (we don't ship + ``narrow_requirement_selection``'s separate promoted set). + + Bench-fixture trigger (2026-05-13, T_PARITY_REAL extras work): + with the extras-roundtrip fix admitting ``protobuf<6`` from + ``sentry-protos`` plus ``protobuf<7,>=4.25`` from google-cloud-*, + a clean-record-first preference thrashed through dozens of + sentry-kafka-schemas / sentry-protos candidates before + backtracking far enough to find the conflict-free + sentry-kafka-schemas 0.1.x line. Promote-to-front matches pip's + behaviour and lets the resolver converge. + """ + + def test_backtrack_causing_identifier_sorts_first(self): + provider = _make_provider() + causing_id = ("alpha", frozenset()) + clean_id = ("beta", frozenset()) + causing_req = _make_requirement(name="alpha", spec=">=1.0") + clean_req = _make_requirement(name="beta", spec=">=1.0") + # ``alpha`` shows up 3x in backtrack_causes; ``beta`` 0x. + backtrack_causes = ( + _ri(causing_req), + _ri(causing_req), + _ri(causing_req), + ) + information = { + causing_id: iter([_ri(causing_req)]), + clean_id: iter([_ri(clean_req)]), + } + pref_causing = provider.get_preference( + causing_id, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=backtrack_causes, + ) + information = { + causing_id: iter([_ri(causing_req)]), + clean_id: iter([_ri(clean_req)]), + } + pref_clean = provider.get_preference( + clean_id, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=backtrack_causes, + ) + # Backtrack-causing identifier sorts BEFORE the clean one. + assert pref_causing < pref_clean + + +class TestGetPreferenceLexicographicTieBreak: + """Bonus bullet: identifiers that tie on every other axis fall + back to alphabetical name order — same as pip's tuple's trailing + ``identifier`` component. Stable order across runs is what makes + lockfile output deterministic (the Q-C parity gate's eventual + requirement).""" + + def test_alphabetical_order_on_full_tie(self): + provider = _make_provider() + id_a = ("alpha", frozenset()) + id_b = ("beta", frozenset()) + req_a = _make_requirement(name="alpha", spec=">=1.0") + req_b = _make_requirement(name="beta", spec=">=1.0") + information = { + id_a: iter([_ri(req_a)]), + id_b: iter([_ri(req_b)]), + } + pref_a = provider.get_preference( + id_a, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=(), + ) + information = { + id_a: iter([_ri(req_a)]), + id_b: iter([_ri(req_b)]), + } + pref_b = provider.get_preference( + id_b, + resolutions={}, + candidates={}, + information=information, + backtrack_causes=(), + ) + assert pref_a < pref_b + + +class TestGetPreferenceEmptyInformation: + """Defensive: ``get_preference`` is callable with no information for + the identifier (``resolvelib`` can do this transiently between + state transitions). Should not raise, should return a sortable + tuple whose pinned / direct flags reflect "unknown" (treated as + not-pinned / not-direct, matching pip's ``has_information=False`` + branch).""" + + def test_no_information_returns_sortable_tuple(self): + provider = _make_provider() + identifier = ("alpha", frozenset()) + # Empty iterator — same shape pip's ``has_information`` branch + # protects against. + pref = provider.get_preference( + identifier, + resolutions={}, + candidates={}, + information={identifier: iter([])}, + backtrack_causes=(), + ) + # Just confirm we got a tuple back that's orderable against + # another preference tuple (same shape). + other_id = ("beta", frozenset()) + other_req = _make_requirement(name="beta", spec="==1.0") + other_pref = provider.get_preference( + other_id, + resolutions={}, + candidates={}, + information={other_id: iter([_ri(other_req)])}, + backtrack_causes=(), + ) + # Comparison must succeed (orderable). The pinned-beta sorts + # before the unknown-alpha. + assert other_pref < pref + + +# --------------------------------------------------------------------------- +# T6 — is_satisfied_by (final-acceptance predicate) +# --------------------------------------------------------------------------- +# +# Per plan T6 + design §5.3, ``is_satisfied_by`` answers the question +# "is this chosen candidate a valid satisfaction of this requirement?" +# Three checks: +# +# 1. ``candidate.version in requirement.specifier`` (prereleases mirror +# T4's policy). +# 2. Extras compatibility — every extra requested by the requirement +# must be in the candidate's advertised ``provides_extras``. Phase 3 +# decision: when ``provides_extras`` is unknown (lazy-metadata: T7's +# ``get_dependencies`` is what would populate it), we mirror pip's +# behaviour and treat the candidate as satisfying. Pip's +# :meth:`SpecifierRequirement.is_satisfied_by` +# (``pipenv/patched/pip/_internal/resolution/resolvelib/requirements.py:111``) +# delegates to ``spec.contains(candidate.version, prereleases=True)`` +# only — extras are gated upstream by the ``(name, extras)`` identifier +# grouping in :meth:`identify` so the predicate never sees a mismatched +# pair. We adopt the same "assume true if extras data not available" +# stance: tested here via a Candidate whose ``extras`` is the default +# empty frozenset. When metadata IS available (Candidate populated +# with a non-empty ``extras``), we strict-check. +# 3. Marker evaluation against ``target_env`` (override of vendored +# :func:`packaging.markers.default_environment`). Requirement with no +# marker passes unconditionally. + + +def _candidate_for_satisfies( + *, + name: str = "django", + version: str = "4.0.1", + extras: frozenset[str] = frozenset(), +): + """Build a :class:`Candidate` for the T6 ``is_satisfied_by`` tests + with just the four fields the predicate reads (``name``, ``version``, + ``extras`` for the extras-compat path). Wraps :func:`_cand` for + readability at the call sites.""" + return _cand(name=name, version=version, extras=extras) + + +class TestIsSatisfiedByVersion: + """Plan T6 bullet 1+2: version-specifier check. + + * ``==4.0.1`` requirement, candidate version ``4.0.1`` → ``True``. + * ``==4.0.1`` requirement, candidate version ``4.0.2`` → ``False``. + """ + + def test_exact_pin_matches_same_version(self): + provider = _make_provider() + req = _make_requirement(name="django", spec="==4.0.1") + cand = _candidate_for_satisfies(name="django", version="4.0.1") + assert provider.is_satisfied_by(req, cand) is True + + def test_exact_pin_rejects_different_version(self): + provider = _make_provider() + req = _make_requirement(name="django", spec="==4.0.1") + cand = _candidate_for_satisfies(name="django", version="4.0.2") + assert provider.is_satisfied_by(req, cand) is False + + def test_range_specifier_accepts_in_range(self): + provider = _make_provider() + req = _make_requirement(name="django", spec=">=4.0,<5.0") + cand = _candidate_for_satisfies(name="django", version="4.2.3") + assert provider.is_satisfied_by(req, cand) is True + + def test_range_specifier_rejects_out_of_range(self): + provider = _make_provider() + req = _make_requirement(name="django", spec=">=4.0,<5.0") + cand = _candidate_for_satisfies(name="django", version="5.1.0") + assert provider.is_satisfied_by(req, cand) is False + + def test_empty_specifier_accepts_any_version(self): + """A Pipfile ``"*"`` entry flattens to an empty + :class:`SpecifierSet` (per T1's ``from_pipfile_entry`` contract), + which should admit any non-prerelease version.""" + provider = _make_provider() + req = _make_requirement(name="django", spec="") + cand = _candidate_for_satisfies(name="django", version="4.2.0") + assert provider.is_satisfied_by(req, cand) is True + + def test_prerelease_candidate_admitted_at_predicate(self): + """Pip parity: :meth:`is_satisfied_by` passes + ``prereleases=True`` unconditionally, because + :meth:`find_matches` (the ``PackageFinder`` analog) has already + applied the prerelease policy upstream. A prerelease arriving + at the predicate IS satisfying as far as the specifier check + goes — re-filtering here would silently reject candidates the + resolver legitimately matched. Mirrors pip's + ``SpecifierRequirement.is_satisfied_by`` at + ``pipenv/patched/pip/_internal/resolution/resolvelib/requirements.py:121``. + """ + provider = _make_provider(allow_prereleases=False) + req = _make_requirement(name="django", spec=">=4.0") + cand = _candidate_for_satisfies(name="django", version="5.0.0a1") + assert provider.is_satisfied_by(req, cand) is True + + +class TestIsSatisfiedByMarker: + """Plan T6 bullet 3: marker evaluation. + + * ``requirement.marker = Marker("python_version < '3.10'")`` + + ``target_env = {"python_version": "3.12"}`` → ``False``. + * Marker evaluates True against target env → predicate returns True. + * Requirement with ``marker is None`` → predicate is marker-blind. + """ + + def test_marker_evaluating_false_rejects(self): + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.markers import Marker + from pipenv.vendor.packaging.specifiers import SpecifierSet + + req = Requirement( + name="django", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=Marker("python_version < '3.10'"), + source="pipfile", + parent=None, + ) + provider = _make_provider(target_env={"python_version": "3.12"}) + cand = _candidate_for_satisfies(name="django", version="4.0.1") + assert provider.is_satisfied_by(req, cand) is False + + def test_marker_evaluating_true_accepts(self): + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.markers import Marker + from pipenv.vendor.packaging.specifiers import SpecifierSet + + req = Requirement( + name="django", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=Marker("python_version >= '3.10'"), + source="pipfile", + parent=None, + ) + provider = _make_provider(target_env={"python_version": "3.12"}) + cand = _candidate_for_satisfies(name="django", version="4.0.1") + assert provider.is_satisfied_by(req, cand) is True + + def test_no_marker_passes_unconditionally(self): + provider = _make_provider(target_env={"python_version": "3.6"}) + req = _make_requirement(name="django", spec="==4.0.1") # marker=None + cand = _candidate_for_satisfies(name="django", version="4.0.1") + assert provider.is_satisfied_by(req, cand) is True + + def test_marker_evaluation_overrides_default_environment(self): + """The marker must be evaluated against ``self._target_env`` — + NOT the running Python's environment. We pin + ``python_version = "3.7"`` in the target env and verify the + marker uses that even though CI is running on a newer Python.""" + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.markers import Marker + from pipenv.vendor.packaging.specifiers import SpecifierSet + + # ``python_version < '3.8'`` is True under the synthetic + # target_env but typically False under the running interpreter + # (CI runs >=3.9). If the impl leaked default_environment, the + # assertion below would flip. + req = Requirement( + name="django", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=Marker("python_version < '3.8'"), + source="pipfile", + parent=None, + ) + provider = _make_provider(target_env={"python_version": "3.7"}) + cand = _candidate_for_satisfies(name="django", version="4.0.1") + assert provider.is_satisfied_by(req, cand) is True + + def test_strip_extras_helper_drops_extra_clause(self): + """T_PARITY_REAL bench regression (project 01 — ``psycopg[binary]``). + + Transitive requirements emitted by T7's + :meth:`get_dependencies` get their ``extra == X`` clauses + stripped from the runtime marker via :func:`_strip_extra_clauses`. + Otherwise re-evaluating the marker in :meth:`is_satisfied_by` + against the plain target env (no ``extra`` key) would collapse + the clause to False and reject every candidate — exactly the + ``google-api-core[grpc]`` → ``grpcio<2.0 ; ... and extra == "grpc"`` + cascade that hung the sentry-base bench fixture. + + The strip preserves all non-extras clauses + (``python_version``, ``sys_platform``, etc.), so install-time + gates still fire. + """ + from pipenv.resolver.pure_python_provider import _strip_extra_clauses + from pipenv.vendor.packaging.markers import Marker + + # Compound marker — extras clause stripped, python_version kept. + stripped = _strip_extra_clauses( + Marker('python_version >= "3.11" and extra == "grpc"') + ) + assert stripped is not None + assert str(stripped) == 'python_version >= "3.11"' + + # Single extras clause — entire marker collapses to None. + assert ( + _strip_extra_clauses(Marker('extra == "binary"')) is None + ) + + # Marker with no extras clause — returned unchanged + # (semantically; the helper re-parses so identity differs). + unchanged = _strip_extra_clauses( + Marker('python_version >= "3.11"') + ) + assert unchanged is not None + assert str(unchanged) == 'python_version >= "3.11"' + + +class TestIsSatisfiedByExtras: + """Plan T6 bullet 2 — extras compatibility under lazy metadata. + + Phase-3 decision recorded on the T6 plan entry: when the candidate's + advertised ``extras`` is empty (the default — lazy metadata not yet + loaded, since ``is_satisfied_by`` runs BEFORE T7's + ``get_dependencies`` fetches METADATA), we mirror pip's stance and + return ``True`` rather than failing a candidate the resolver has + legitimately matched via the ``(name, extras)`` identifier grouping + (pip: + ``pipenv/patched/pip/_internal/resolution/resolvelib/requirements.py:111`` + — pip checks only the specifier). When the candidate DOES advertise + a non-empty extras set, we strict-check — extras-bearing candidates + are typically synthesised by tests / future code paths and a + mismatch there is informative. + """ + + def test_extras_unknown_lazy_metadata_returns_true(self): + """Candidate's ``extras`` is the default empty frozenset — we + treat this as "metadata not yet loaded" and admit the candidate. + Pinned behaviour: requirement asks for ``[argon2]`` extra, + candidate exposes no extras → still ``True``.""" + provider = _make_provider() + req = _make_requirement( + name="django", + spec=">=4.0", + extras=frozenset({"argon2"}), + ) + cand = _candidate_for_satisfies(name="django", version="4.2.0") + # Default ``extras=frozenset()`` on the candidate models the + # lazy-metadata case. + assert cand.extras == frozenset() + assert provider.is_satisfied_by(req, cand) is True + + def test_extras_satisfied_when_candidate_advertises_superset(self): + provider = _make_provider() + req = _make_requirement( + name="django", + spec=">=4.0", + extras=frozenset({"argon2"}), + ) + cand = _candidate_for_satisfies( + name="django", + version="4.2.0", + extras=frozenset({"argon2", "bcrypt"}), + ) + assert provider.is_satisfied_by(req, cand) is True + + def test_extras_rejected_when_candidate_advertises_disjoint(self): + """Strict check: when the candidate DOES advertise extras (so + metadata is loaded) and the requested extra isn't in the set, + the predicate returns False.""" + provider = _make_provider() + req = _make_requirement( + name="django", + spec=">=4.0", + extras=frozenset({"argon2"}), + ) + cand = _candidate_for_satisfies( + name="django", + version="4.2.0", + extras=frozenset({"bcrypt"}), + ) + assert provider.is_satisfied_by(req, cand) is False + + def test_no_extras_requested_passes_regardless(self): + provider = _make_provider() + req = _make_requirement( + name="django", spec=">=4.0", extras=frozenset() + ) + cand = _candidate_for_satisfies( + name="django", + version="4.2.0", + extras=frozenset({"bcrypt"}), + ) + assert provider.is_satisfied_by(req, cand) is True + + +# --------------------------------------------------------------------------- +# T7 — get_dependencies (Q-A fail-loud for sdist; Requires-Dist parsing for +# wheels; marker filtering with parent-extras context) +# --------------------------------------------------------------------------- +# +# Validation matrix (per ``initiative-g-phase3-plan.md`` T7 + design §5.3, +# updated by Initiative G Phase 3b T_S3): +# +# 1. Wheel candidate with ``Requires-Dist: numpy>=1.20,<2.0`` → +# ``[Requirement(name="numpy", specifier=">=1.20,<2.0", +# source="transitive", parent=)]``. +# 2. Wheel candidate with ``Requires-Dist: pytest; extra=='dev'`` and the +# candidate did NOT request the ``[dev]`` extra → marker evaluates +# False against the empty-extras context; req filtered out. +# 3. Wheel candidate with ``Requires-Dist: pytest; extra=='dev'`` and +# the candidate DID request the ``[dev]`` extra → marker evaluates +# True under ``{"extra": "dev"}``; req kept. +# 4. Sdist candidate (``.tar.gz`` filename) → routes through +# ``self._metadata_fetcher`` like a wheel does; the stubbed fetcher +# returns a synthetic :class:`CoreMetadata` and the transitive +# :class:`Requirement` set is returned. (Phase 3a raised +# ``_SdistEncountered`` here; Phase 3b T_S3 removed that gate now +# that T_S2 routes sdists through T_S1's PEP 517 builder.) +# +# Implementation reference (pip's analog, cited in production code): +# ``pipenv/patched/pip/_internal/metadata/importlib/_dists.py:224`` — +# ``BaseDistribution.iter_dependencies(extras)`` — three-branch logic: +# no-marker → yield; no requested extras + ``marker.evaluate({"extra": +# ""})`` → yield; else yield iff any ``{"extra": e}`` makes the marker +# True. + + +def _metadata( + *, + name: str = "django", + version: str = "4.2.0", + requires_dist: tuple[str, ...] = (), + requires_python: str | None = None, + provides_extras: frozenset[str] = frozenset(), + summary: str | None = None, +): + """Build a :class:`CoreMetadata` for T7 tests. + + Wraps :class:`pipenv.resolver.pure_python_metadata.CoreMetadata` so + each test names only the field it cares about; the rest take inert + defaults. + """ + from pipenv.resolver.pure_python_metadata import CoreMetadata + + return CoreMetadata( + name=name, + version=version, + requires_python=requires_python, + requires_dist=requires_dist, + provides_extras=provides_extras, + summary=summary, + ) + + +def _metadata_fetcher_stub(mapping: dict): + """Return a ``metadata_fetcher`` callable suitable for the provider. + + ``mapping`` is ``{candidate_url: CoreMetadata}``. The callable + matches the shape the provider expects: + ``metadata_fetcher(candidate) -> CoreMetadata``. T9 builds the + real wiring around :func:`fetch_metadata`; tests use this stub + so no HTTP I/O runs. + """ + + def fetch(candidate): + try: + return mapping[candidate.url] + except KeyError as exc: + raise AssertionError( + f"test stub: no metadata seeded for {candidate.url}" + ) from exc + + return fetch + + +class TestGetDependenciesWheelRequiresDist: + """Plan T7 bullet 1: wheel candidate's ``Requires-Dist`` becomes a + list of :class:`Requirement`\\ s with ``source="transitive"`` and + ``parent=``.""" + + def test_single_requires_dist_returns_one_requirement(self): + cand = _cand(name="django", version="4.2.0", is_wheel=True) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=("numpy>=1.20,<2.0",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + + deps = list(provider.get_dependencies(cand)) + + assert len(deps) == 1 + dep = deps[0] + assert dep.name == "numpy" + # SpecifierSet's str form is stable across orderings — compare + # via the underlying spec strings sorted. + assert sorted(str(s) for s in dep.specifier) == sorted( + (">=1.20", "<2.0") + ) + assert dep.source == "transitive" + assert dep.parent == "django" + assert dep.marker is None + + def test_multiple_requires_dist_returns_all(self): + cand = _cand(name="flask", version="2.3.0", is_wheel=True) + meta = _metadata( + name="flask", + version="2.3.0", + requires_dist=("werkzeug>=2.3", "jinja2>=3.0"), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + + deps = list(provider.get_dependencies(cand)) + names = sorted(d.name for d in deps) + assert names == ["jinja2", "werkzeug"] + assert all(d.source == "transitive" for d in deps) + assert all(d.parent == "flask" for d in deps) + + def test_empty_requires_dist_returns_empty_list(self): + """A wheel with no ``Requires-Dist`` headers — a leaf in the + graph — must yield zero requirements (NOT raise).""" + cand = _cand(name="six", version="1.16.0", is_wheel=True) + meta = _metadata( + name="six", version="1.16.0", requires_dist=() + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + assert list(provider.get_dependencies(cand)) == [] + + +class TestGetDependenciesMarkerFilter: + """Plan T7 bullet 2 + 3: marker evaluation with parent-extras context. + + Mirrors pip's three-branch logic at + ``pipenv/patched/pip/_internal/metadata/importlib/_dists.py:224``: + + * No marker -> yield. + * Marker present + no parent extras -> yield iff + ``marker.evaluate({"extra": ""})`` is True. + * Marker present + parent extras non-empty -> yield iff + ``marker.evaluate({"extra": e})`` is True for any e in extras. + """ + + def test_extra_marker_filtered_out_when_parent_has_no_extras(self): + """Plan T7 bullet 2: ``Requires-Dist: pytest; extra=='dev'`` + with parent candidate's ``extras == frozenset()`` -> marker + evaluates False under ``{"extra": ""}``; req filtered out.""" + cand = _cand( + name="django", + version="4.2.0", + is_wheel=True, + extras=frozenset(), # parent did NOT ask for [dev] + ) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=("pytest ; extra == 'dev'",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + deps = list(provider.get_dependencies(cand)) + assert deps == [] + + def test_extra_marker_kept_when_parent_requested_that_extra(self): + """Plan T7 bullet 3 (active-extras case): + ``Requires-Dist: pytest; extra=='dev'`` with parent candidate's + ``extras == frozenset({"dev"})`` -> marker evaluates True under + ``{"extra": "dev"}``; req kept with ``parent=``. + """ + cand = _cand( + name="django", + version="4.2.0", + is_wheel=True, + extras=frozenset({"dev"}), # parent DID ask for [dev] + ) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=("pytest>=7 ; extra == 'dev'",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + deps = list(provider.get_dependencies(cand)) + # When parent_extras is non-empty, get_dependencies emits a + # synthetic base-version pin FIRST (mirrors pip's + # :class:`ExtrasCandidate.iter_dependencies`) — strip it for + # the assertion below, which is about the marker-gated + # transitive. + non_pin = [d for d in deps if d.name != "django"] + assert len(non_pin) == 1 + assert non_pin[0].name == "pytest" + assert non_pin[0].parent == "django" + assert non_pin[0].source == "transitive" + # Runtime marker is None: the ``extra == "dev"`` clause was + # already consumed at emission time (the gate that made this + # requirement survive :meth:`_marker_active_for_extras`), and + # carrying it onto :attr:`Requirement.marker` would make T6's + # :meth:`is_satisfied_by` re-evaluate it against the plain + # target env (no ``extra`` key) and reject every pytest + # candidate. ``introducing_marker`` keeps the original for + # the lockfile emitter (T_M3). + assert non_pin[0].marker is None + assert non_pin[0].introducing_marker is not None + assert str(non_pin[0].introducing_marker) == 'extra == "dev"' + # Synthetic base-pin: ``django==4.2.0`` with no extras. + pin = next((d for d in deps if d.name == "django"), None) + assert pin is not None + assert str(pin.specifier) == "==4.2.0" + assert pin.extras == frozenset() + assert pin.marker is None + + def test_non_extra_marker_evaluated_against_target_env(self): + """A non-extra marker (e.g. ``python_version < '3.8'``) is + evaluated against the provider's ``target_env`` overlay -- pinning + ``python_version = "3.12"`` filters a ``< '3.8'`` marker out.""" + cand = _cand( + name="django", version="4.2.0", is_wheel=True + ) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=( + "typing_extensions ; python_version < '3.8'", + "sqlparse>=0.3", + ), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + deps = list(provider.get_dependencies(cand)) + names = [d.name for d in deps] + # typing_extensions filtered out -- its marker is False under + # python_version=3.12. sqlparse survives (no marker). + assert names == ["sqlparse"] + + +class TestGetDependenciesSdistRoutesThroughMetadataFetcher: + """Phase 3b T_S3: ``get_dependencies`` no longer raises + :class:`_SdistEncountered` on a non-wheel candidate. Instead it + routes the candidate through ``self._metadata_fetcher`` exactly + like a wheel candidate — T_S2's + :meth:`MetadataFetcher.fetch_metadata` is responsible for branching + on ``candidate.is_wheel`` and routing sdists through T_S1's PEP + 517 builder. + + These tests stub the metadata fetcher in-memory so no PEP 517 + build runs; the assertion is that the provider treats sdist and + wheel candidates symmetrically at the ``get_dependencies`` layer. + """ + + def test_sdist_candidate_routes_through_metadata_fetcher(self): + """An sdist candidate's transitives flow through the stubbed + metadata fetcher the same way a wheel's would — no exception + is raised; the returned :class:`Requirement` objects mirror + the ``Requires-Dist`` entries supplied via the stub. + """ + cand = _cand( + name="legacy", version="0.1", is_wheel=False + ) + # The stub returns a synthetic CoreMetadata as if T_S1 had + # built the sdist on the fly. + meta = _metadata( + name="legacy", + version="0.1", + requires_dist=("six>=1.16",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + + deps = list(provider.get_dependencies(cand)) + + assert len(deps) == 1 + dep = deps[0] + assert dep.name == "six" + assert dep.source == "transitive" + assert dep.parent == "legacy" + # SpecifierSet stringification — pin the spec set explicitly so + # a future packaging-version bump that reorders the repr can't + # silently regress the assertion. + assert str(dep.specifier) == ">=1.16" + + def test_sdist_invokes_metadata_fetcher_exactly_once(self): + """The provider must consult the fetcher for an sdist candidate + — pinning the post-T_S3 contract that there is no longer a + short-circuit gate before the call. + """ + cand = _cand( + name="legacy", version="0.1", is_wheel=False + ) + + call_log: list = [] + + def recording_fetcher(c): + call_log.append(c) + from pipenv.resolver.pure_python_metadata import CoreMetadata + + return CoreMetadata( + name="legacy", + version="0.1", + requires_python=None, + requires_dist=(), + provides_extras=frozenset(), + summary=None, + ) + + provider = _make_provider( + metadata_fetcher=recording_fetcher, + target_env={"python_version": "3.12"}, + ) + + deps = list(provider.get_dependencies(cand)) + # Empty Requires-Dist → no transitives. + assert deps == [] + # The fetcher fired exactly once, with the sdist candidate. + assert len(call_log) == 1 + assert call_log[0] is cand + + def test_sdist_encountered_still_importable(self): + """Back-compat pin: the :class:`_SdistEncountered` class is no + longer raised from production code (T_S3) but remains + importable for external test suites that referenced the + Phase 3a contract. See its docstring for the deprecation + notice. + """ + from pipenv.resolver.pure_python_provider import _SdistEncountered + + assert issubclass(_SdistEncountered, Exception) + # The constructor still accepts a candidate object — preserves + # the Phase 3a public shape so legacy ``try/except`` blocks + # that catch + introspect the exception keep working. + instance = _SdistEncountered(_cand(name="x", version="0", is_wheel=False)) + assert instance.candidate.name == "x" + + +class TestGetDependenciesMalformedRequiresDist: + """Defensive: a malformed ``Requires-Dist`` entry (parser raises) + surfaces as an error rather than silently dropping the dep. + + Pip's ``iter_dependencies`` at + ``pipenv/patched/pip/_internal/metadata/importlib/_dists.py:224`` + propagates ``InvalidRequirement`` from ``packaging`` -- we mirror. + """ + + def test_invalid_requires_dist_raises_invalid_requirement(self): + from pipenv.vendor.packaging.requirements import InvalidRequirement + + cand = _cand( + name="django", version="4.2.0", is_wheel=True + ) + meta = _metadata( + name="django", + version="4.2.0", + # ``!@#`` is not a legal requirement spec -- parser raises. + requires_dist=("!@# not a real spec",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + + try: + list(provider.get_dependencies(cand)) + except InvalidRequirement: + pass # expected + else: # pragma: no cover - acceptance criterion + raise AssertionError( + "expected InvalidRequirement from malformed Requires-Dist" + ) + + +# --------------------------------------------------------------------------- +# T13 — Edge-case tests added to push coverage on +# ``pipenv/resolver/pure_python_provider.py`` to >= 90 %. +# +# Audit list (per plan T13): +# +# * ``identify``: duck-typed Candidate subclass without ``extras`` field +# falls back to ``frozenset()``. +# * ``find_matches``: +# - Dedup by ``(name, version, filename)`` removes a duplicate. +# - Cache miss across multiple index_urls, with no ``populate`` +# attribute on the fetcher (defensive ``getattr``). +# - ``InvalidVersion`` candidate filtered out from satisfaction +# check. +# - ``allow_prereleases=True`` admits a prerelease against a +# specifier that does not opt in. +# - ``requires_python`` target_python ``None`` returns True. +# - ``InvalidSpecifier`` in candidate's ``requires_python`` is +# permissively accepted. +# - ``requires_python``'s ``SpecifierSet.contains`` raising +# ``InvalidVersion`` is permissively accepted. +# * ``get_preference``: +# - A row whose ``requirement.specifier`` is ``None`` (duck-typed +# requirement without a SpecifierSet) is tolerated. +# - ``backtrack_causes`` entry whose ``requirement`` is ``None`` is +# skipped. +# - ``backtrack_causes`` entry whose ``self.identify`` raises is +# skipped (defensive try/except). +# - ``backtrack_causes`` entry for a DIFFERENT identifier doesn't +# bump the count. +# * ``is_satisfied_by``: +# - Candidate version unparseable -> False (loud-failure stance). +# - Marker that raises during ``.evaluate(...)`` -> False (defensive +# except). +# * ``get_dependencies``: +# - Empty / whitespace-only ``Requires-Dist`` line is skipped. +# - Active extra: a malformed marker under one extra context is +# skipped (defensive ``except`` in the parent_extras branch). +# - Empty parent_extras + malformed marker is treated as inactive. + + +class TestIdentifyDuckTypedCandidate: + """``identify`` accepts any object exposing ``.name`` and + optionally ``.extras``. The fallback path uses + ``getattr(..., "extras", frozenset())`` -- pin it explicitly with a + duck-typed object that doesn't even have an ``extras`` attribute.""" + + def test_duck_typed_candidate_without_extras_falls_back_to_frozenset(self): + provider = _make_provider() + + class _DuckCandidate: + name = "django" + + ident = provider.identify(_DuckCandidate()) + assert ident == ("django", frozenset()) + + +class TestFindMatchesDedup: + """Plan T13: dedup on ``(name, version, filename)`` collapses + duplicate entries (same wheel served by mirror indexes). Two cache + entries that share the identity tuple should yield ONE result.""" + + def test_duplicate_candidates_collapsed(self): + # Two cache hits for the same package + version + filename -- + # the second occurrence triggers the ``continue`` branch on + # ``key in seen``. + c1 = _cand(name="django", version="4.2.0") + c2 = _cand(name="django", version="4.2.0") # identical key + c3 = _cand(name="django", version="4.1.0") + cache = _FakeCache({(_INDEX, "django"): (c1, c2, c3)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + versions = [c.version for c in result] + # Two unique versions after dedup; high-version first. + assert versions == ["4.2.0", "4.1.0"] + + +class TestFindMatchesFetcherWithoutPopulate: + """Defensive: ``find_matches`` uses ``getattr(fetcher, "populate", None)`` + so a fetcher object without a ``populate`` attribute (legitimate in + smoke tests) doesn't crash on cache miss.""" + + def test_no_populate_attribute_returns_empty_without_raise(self): + cache = _FakeCache() # empty -- triggers populate path + # Use a plain object() as the fetcher -- no ``populate`` method. + provider = _make_provider( + cache=cache, fetcher=object(), index_urls=[_INDEX] + ) + + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + assert result == [] + + +class TestFindMatchesInvalidVersionCandidate: + """A candidate whose ``.version`` cannot be parsed by + :class:`packaging.version.Version` is filtered out at the + satisfaction check (loud-failure stance shared with T4 + T6).""" + + def test_unparseable_version_excluded(self): + # Mix a valid candidate with one whose version is junk. The + # valid candidate sails through; the junk one is dropped by the + # ``InvalidVersion`` branch in + # ``_candidate_satisfies_requirements``. + good = _cand(name="django", version="4.2.0") + bad = _cand(name="django", version="not-a-version") + cache = _FakeCache({(_INDEX, "django"): (good, bad)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + versions = [c.version for c in result] + assert versions == ["4.2.0"] + + +class TestFindMatchesAllowPrereleases: + """``allow_prereleases=True`` admits a prerelease version even when + the specifier itself does not opt in (e.g. plain ``>=4.0``).""" + + def test_allow_prereleases_admits_prerelease(self): + cands = [ + _cand(name="django", version="4.2.0"), + _cand(name="django", version="5.0.0a1"), + ] + cache = _FakeCache({(_INDEX, "django"): tuple(cands)}) + provider = _make_provider( + cache=cache, + index_urls=[_INDEX], + allow_prereleases=True, + ) + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + versions = [c.version for c in result] + # Prerelease appears at the head (highest by Version order). + assert "5.0.0a1" in versions + assert "4.2.0" in versions + + def test_specifier_with_prerelease_opt_in_admits_prerelease(self): + """When the specifier itself opts in (e.g. ``>=5.0a0``), the + provider's :meth:`_candidate_satisfies_requirements` passes + ``prereleases=True`` to :meth:`SpecifierSet.contains` regardless + of the ``allow_prereleases`` flag. Pins the + ``spec.prereleases`` branch of the prerelease policy.""" + cands = [ + _cand(name="django", version="5.0.0a1"), + _cand(name="django", version="5.0.0a2"), + ] + cache = _FakeCache({(_INDEX, "django"): tuple(cands)}) + provider = _make_provider( + cache=cache, + index_urls=[_INDEX], + allow_prereleases=False, + ) + identifier = ("django", frozenset()) + # The ``>=5.0a0`` specifier flips its own ``prereleases`` to True. + req = _make_requirement(name="django", spec=">=5.0a0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + versions = [c.version for c in result] + assert versions == ["5.0.0a2", "5.0.0a1"] + + +class TestFindMatchesPrereleaseStrictExclusion: + """Bench-fixture regression: a plain ``>=X`` specifier must NOT + admit prereleases when stable versions matching the constraint are + also available. + + Pip mirrors PEP 440 §"Pre-releases" via + :meth:`SpecifierSet.filter` over the FULL candidate list (so the + "no final release matched" fallback fires only when zero stables + pass). Pure-python's per-candidate + :meth:`SpecifierSet.contains(v, prereleases=None)` short-circuited + on a one-element iterable inside ``filter`` and silently admitted + every prerelease — picking ``billiard 4.3.0rc1``, + ``hiredis 3.4.0.dev0``, and ``sentry-sdk 3.0.0a7`` instead of the + stables pip selected on the same bench fixture. + """ + + def test_prerelease_excluded_when_stable_matches(self): + cands = [ + _cand(name="billiard", version="4.2.4"), + _cand(name="billiard", version="4.3.0rc1"), + ] + cache = _FakeCache({(_INDEX, "billiard"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + identifier = ("billiard", frozenset()) + # Plain stable spec — no prerelease opt-in. + req = _make_requirement(name="billiard", spec=">=4.2.1,<5.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + versions = [c.version for c in result] + # 4.3.0rc1 is filtered out — pip's strict-prerelease pass. + assert versions == ["4.2.4"] + + def test_pep440_fallback_when_only_prereleases_available(self): + # Mirrors pip's ``get_applicable_candidates`` fallback: when + # the strict pass yields zero candidates, retry admitting + # prereleases. Trigger: a package with only prerelease + # artifacts at or above the lower bound. + cands = [ + _cand(name="alpha-only", version="3.0.0a1"), + _cand(name="alpha-only", version="3.0.0a2"), + ] + cache = _FakeCache({(_INDEX, "alpha-only"): tuple(cands)}) + provider = _make_provider(cache=cache, index_urls=[_INDEX]) + identifier = ("alpha-only", frozenset()) + # ``>=2.5`` is a plain stable spec but both prereleases satisfy + # it numerically (``3.0.0a1 >= 2.5``). Strict pass yields + # nothing; fallback admits the prereleases. + req = _make_requirement(name="alpha-only", spec=">=2.5") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + versions = [c.version for c in result] + assert versions == ["3.0.0a2", "3.0.0a1"] + + +class TestFindMatchesRequiresPythonEdgeCases: + """Cover the three permissive branches of + ``_candidate_requires_python_ok``: + + 1. ``target_python`` is ``None`` -> accept regardless of + ``requires_python``. + 2. ``requires_python`` is malformed (``InvalidSpecifier``) -> accept. + 3. ``SpecifierSet.contains`` raises ``InvalidVersion`` -> accept. + + All three are explicit "don't make the package invisible because of + a tiny upstream data oddity" branches (mirror pip's + :func:`evaluate_link`). + """ + + def test_no_target_python_accepts_candidate(self): + # ``target_env`` has no ``python_version`` key -> target_python + # is None -> requires_python check short-circuits to True. + cand = _cand( + name="django", version="4.2.0", requires_python=">=3.10" + ) + cache = _FakeCache({(_INDEX, "django"): (cand,)}) + provider = _make_provider( + cache=cache, + index_urls=[_INDEX], + target_env={}, # no python_version key + ) + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + assert [c.version for c in result] == ["4.2.0"] + + def test_invalid_specifier_in_requires_python_accepts(self): + """A garbage ``requires_python`` string raises + :class:`InvalidSpecifier` inside the helper; the helper falls + through to accept (mirrors pip's behaviour).""" + cand = _cand( + name="django", + version="4.2.0", + requires_python="not-a-specifier", + ) + cache = _FakeCache({(_INDEX, "django"): (cand,)}) + provider = _make_provider( + cache=cache, + index_urls=[_INDEX], + target_env={"python_version": "3.12"}, + ) + identifier = ("django", frozenset()) + req = _make_requirement(name="django", spec=">=4.0") + result = list( + provider.find_matches( + identifier, + requirements={identifier: iter([req])}, + incompatibilities={}, + ) + ) + assert [c.version for c in result] == ["4.2.0"] + + # Note on the ``InvalidVersion`` branch at lines 429-430 of + # ``pure_python_provider.py``: in the current vendored + # ``packaging`` version, :meth:`SpecifierSet.contains` does NOT + # raise :class:`InvalidVersion` -- it returns ``False`` for + # malformed input. The branch is documented defensive scaffolding + # against a hypothetical future ``packaging`` behaviour change, so + # we cannot exercise it from a test without monkey-patching the + # vendored module. Per the T13 plan that branch is deliberately + # left uncovered; total coverage (98 %+) is well above the 90 % + # gate. + + +class TestGetPreferenceDefensiveBranches: + """Edge cases inside ``get_preference``: + + * A ``RequirementInformation`` row whose ``.requirement.specifier`` + attribute is ``None`` (duck-typed shape) is tolerated -- the + operators-loop just ``continue``\\s past it. + * ``backtrack_causes`` entry whose ``.requirement`` is ``None`` is + skipped. + * ``backtrack_causes`` entry for a DIFFERENT identifier doesn't + bump the local count. + * ``backtrack_causes`` entry whose ``self.identify`` raises is + skipped via the defensive try/except. + """ + + def test_requirement_specifier_none_is_tolerated(self): + from pipenv.patched.pip._vendor.resolvelib.structs import ( + RequirementInformation, + ) + + provider = _make_provider() + identifier = ("alpha", frozenset()) + + class _NoSpecifierReq: + name = "alpha" + extras = frozenset() + specifier = None + source = "pipfile" + + row = RequirementInformation( + requirement=_NoSpecifierReq(), parent=None + ) + # Should not raise. + pref = provider.get_preference( + identifier, + resolutions={}, + candidates={}, + information={identifier: iter([row])}, + backtrack_causes=(), + ) + assert isinstance(pref, tuple) + # The ``operators`` list is empty when every row's spec is None, + # so ``is_unfree`` is False (== ``not bool(operators)``). The + # ``not is_unfree`` slot must be True (i.e. comes AFTER an + # operator-bearing requirement). + other_id = ("beta", frozenset()) + other_req = _make_requirement(name="beta", spec=">=1.0") + other_pref = provider.get_preference( + other_id, + resolutions={}, + candidates={}, + information={other_id: iter([_ri(other_req)])}, + backtrack_causes=(), + ) + # ``other`` (has any op) sorts before ``alpha`` (no op). + assert other_pref < pref + + def test_backtrack_cause_with_no_requirement_skipped(self): + from pipenv.patched.pip._vendor.resolvelib.structs import ( + RequirementInformation, + ) + + provider = _make_provider() + identifier = ("alpha", frozenset()) + req = _make_requirement(name="alpha", spec=">=1.0") + # A backtrack-causes row whose ``.requirement`` is ``None`` -- + # defensive ``getattr(row, "requirement", None)`` skips it. + rogue_row = RequirementInformation(requirement=None, parent=None) + pref = provider.get_preference( + identifier, + resolutions={}, + candidates={}, + information={identifier: iter([_ri(req)])}, + backtrack_causes=(rogue_row,), + ) + # ``not has_backtracked`` slot (index 0) must be True -- + # the rogue row didn't bump the count, so the identifier + # stays "clean record" and sorts AFTER any promoted rows. + assert pref[0] is True + + def test_backtrack_cause_for_different_identifier_not_counted(self): + provider = _make_provider() + identifier_a = ("alpha", frozenset()) + identifier_b = ("beta", frozenset()) + req_a = _make_requirement(name="alpha", spec=">=1.0") + req_b = _make_requirement(name="beta", spec=">=1.0") + # backtrack_causes points at ``beta`` -- it must NOT bump + # ``alpha``'s count. + causes = (_ri(req_b), _ri(req_b)) + pref_alpha = provider.get_preference( + identifier_a, + resolutions={}, + candidates={}, + information={identifier_a: iter([_ri(req_a)])}, + backtrack_causes=causes, + ) + # No backtracks for alpha -> slot 0 (``not has_backtracked``) + # is True (clean record, sorts AFTER promoted identifiers). + assert pref_alpha[0] is True + + def test_backtrack_cause_with_unidentifiable_requirement_skipped(self): + """If ``self.identify`` raises on a backtrack-causes row, the + defensive try/except in ``get_preference`` skips the row rather + than crashing the resolve.""" + from pipenv.patched.pip._vendor.resolvelib.structs import ( + RequirementInformation, + ) + + provider = _make_provider() + identifier = ("alpha", frozenset()) + req = _make_requirement(name="alpha", spec=">=1.0") + + # Build a row whose ``.requirement`` doesn't satisfy the + # ``identify`` contract (no ``.name``/``.extras``). ``identify`` + # branches on isinstance(Requirement) and falls through to + # attribute access -- AttributeError gets swallowed. + class _BrokenReq: + pass + + broken_row = RequirementInformation( + requirement=_BrokenReq(), parent=None + ) + # Should NOT raise; backtrack count for ``alpha`` stays 0, + # so slot 0 (``not has_backtracked``) is True. + pref = provider.get_preference( + identifier, + resolutions={}, + candidates={}, + information={identifier: iter([_ri(req)])}, + backtrack_causes=(broken_row,), + ) + assert pref[0] is True + + +class TestIsSatisfiedByVersionUnparseable: + """``is_satisfied_by`` returns False when the candidate's version + can't be parsed by :class:`packaging.version.Version` (loud-failure + stance shared with T4).""" + + def test_unparseable_version_returns_false(self): + provider = _make_provider() + req = _make_requirement(name="django", spec=">=4.0") + cand = _candidate_for_satisfies( + name="django", version="not-a-version" + ) + assert provider.is_satisfied_by(req, cand) is False + + +class TestIsSatisfiedByMarkerEvaluationError: + """When ``marker.evaluate(env)`` raises (e.g. references an unknown + marker variable in the overlay), ``is_satisfied_by`` returns False + rather than crashing -- defensive ``except``.""" + + def test_marker_evaluation_raises_returns_false(self): + from pipenv.resolver.pure_python_requirement import Requirement + from pipenv.vendor.packaging.specifiers import SpecifierSet + + class _RaisingMarker: + def evaluate(self, env): + raise RuntimeError("synthetic marker failure") + + req = Requirement( + name="django", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=_RaisingMarker(), + source="pipfile", + parent=None, + ) + provider = _make_provider(target_env={"python_version": "3.12"}) + cand = _candidate_for_satisfies(name="django", version="4.0.1") + assert provider.is_satisfied_by(req, cand) is False + + +class TestGetDependenciesEmptyOrWhitespaceRequiresDist: + """A blank ``Requires-Dist`` entry (e.g. trailing newline in a + hand-crafted METADATA fixture) is skipped before reaching the + parser, rather than raising ``InvalidRequirement``.""" + + def test_empty_requires_dist_entry_skipped(self): + cand = _cand(name="django", version="4.2.0", is_wheel=True) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=(" ", "", "numpy>=1.20"), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + deps = list(provider.get_dependencies(cand)) + # Only the real entry survives. + assert [d.name for d in deps] == ["numpy"] + + +class TestGetDependenciesMarkerEvaluationException: + """Defensive: a marker that RAISES during evaluation (rather than + returning False) is treated as inactive, both in the + "parent has no extras" branch and the "parent has extras" branch. + + Mirrors the design comment at + ``_marker_active_for_extras`` -- pip silently ignores the same + shape via :meth:`Marker.evaluate`'s missing-key path; we surface + the same outcome (the dep just doesn't apply). + """ + + class _RaisingMarker: + """Marker that always raises -- triggers the ``except`` branch + in both code paths of ``_marker_active_for_extras``.""" + + def evaluate(self, env): + raise RuntimeError("synthetic marker failure") + + def _patch_first_marker(self, monkeypatch, raising_marker): + """Replace the first parsed marker with one that raises. + + ``packaging.requirements.Requirement`` parses the marker on + construction; we override the parsed instance to inject the + raising stub after the fact via a wrapper around + ``PackagingRequirement.__init__``. Cleaner than synthesising + a malformed METADATA line because we need a SPECIFIC failure + mode (raise during evaluate, not parse-time failure). + """ + from pipenv.vendor.packaging import requirements as pkg_reqs + + real_init = pkg_reqs.Requirement.__init__ + + def patched_init(self, requirement_string): + real_init(self, requirement_string) + if self.marker is not None: + self.marker = raising_marker + + monkeypatch.setattr( + pkg_reqs.Requirement, "__init__", patched_init + ) + + def test_marker_raises_with_no_parent_extras_skips_dep( + self, monkeypatch + ): + self._patch_first_marker(monkeypatch, self._RaisingMarker()) + cand = _cand( + name="django", + version="4.2.0", + is_wheel=True, + extras=frozenset(), # no parent extras + ) + meta = _metadata( + name="django", + version="4.2.0", + # Marker present -- triggers the marker-eval branch with + # the patched raising marker. + requires_dist=("pytest ; python_version > '3'",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + deps = list(provider.get_dependencies(cand)) + # The marker raised under {"extra": ""} -- treated as inactive. + assert deps == [] + + def test_marker_raises_with_parent_extras_skips_dep( + self, monkeypatch + ): + self._patch_first_marker(monkeypatch, self._RaisingMarker()) + cand = _cand( + name="django", + version="4.2.0", + is_wheel=True, + extras=frozenset({"dev"}), # parent DOES request extras + ) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=("pytest ; extra == 'dev'",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + deps = list(provider.get_dependencies(cand)) + # Both extras-context evaluations raise -- treated as inactive. + # The only entry that survives is the synthetic base-version + # pin (`django==4.2.0`) emitted because parent_extras is + # non-empty; the marker-gated pytest transitive is dropped. + non_pin = [d for d in deps if d.name != "django"] + assert non_pin == [] + + +class TestGetDependenciesIntroducingMarker: + """Plan T_M2 (Initiative G Phase 3b): when ``get_dependencies`` + builds a transitive :class:`Requirement` from a ``Requires-Dist`` + line, the parser's ``.marker`` is threaded through into the new + ``introducing_marker`` slot (T_M1 addition). T_M3 will read this + field in ``_translate_mapping`` to emit a ``markers=...`` clause + on the lockfile entry; this test class pins the upstream contract. + + Note on the dual marker fields on :class:`Requirement`: + + * ``marker`` — populated by ``Requirement.from_pipfile_entry`` for + top-level Pipfile entries (constraint-side marker) AND, today, + by ``get_dependencies`` for transitives (mirrors the parser's + ``.marker`` directly). The legacy T7 unit-test contract + (``TestGetDependenciesMarkerFilter:: + test_extra_marker_kept_when_parent_requested_that_extra``) pins + this behaviour. + * ``introducing_marker`` — populated here (T_M2); represents the + *Requires-Dist-side* marker that introduced a transitive, read + by T_M3's ``_translate_mapping`` to emit a ``markers=...`` + clause on the lockfile entry. Splitting the field from + ``marker`` preserves the T7 contract while letting T_M3 evolve + independently (e.g. T_M3 may choose to canonicalise / normalise + the marker string differently from how the resolver evaluates + it). + """ + + def test_transitive_with_marker_carries_introducing_marker(self): + """``Requires-Dist: pytest; python_version < '3.10'`` parsed + under target_env ``python_version='3.9'`` (so the + marker-extras-active filter keeps the entry) -> resulting + :class:`Requirement.introducing_marker` equals the parsed + :class:`Marker`.""" + from pipenv.vendor.packaging.markers import Marker + + cand = _cand(name="django", version="4.2.0", is_wheel=True) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=("pytest ; python_version < '3.10'",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + # Pin target_env to a Python version that makes the marker + # evaluate True -- otherwise the marker-extras-active filter + # drops the requirement before we see it. + target_env={"python_version": "3.9"}, + ) + deps = list(provider.get_dependencies(cand)) + assert len(deps) == 1 + dep = deps[0] + # introducing_marker carries the parsed marker (T_M2 contract). + assert dep.introducing_marker is not None + # Marker comparison: ``packaging``'s ``Marker.__str__`` is + # stable for the canonical form -- compare via string round-trip. + expected = Marker("python_version < '3.10'") + assert str(dep.introducing_marker) == str(expected) + + def test_transitive_without_marker_has_introducing_marker_none(self): + """``Requires-Dist: numpy>=1.20`` (no marker clause) -> the + resulting :class:`Requirement.introducing_marker` is ``None``.""" + cand = _cand(name="django", version="4.2.0", is_wheel=True) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=("numpy>=1.20",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + deps = list(provider.get_dependencies(cand)) + assert len(deps) == 1 + dep = deps[0] + assert dep.name == "numpy" + assert dep.introducing_marker is None + + def test_marker_extras_filter_unaffected(self): + """``Requires-Dist: pytest; extra=='dev'`` with parent + ``extras=frozenset()`` still drops the requirement -- the + T_M2 addition is strictly additive and must not bypass the + existing T7 marker-extras-active filter.""" + cand = _cand( + name="django", + version="4.2.0", + is_wheel=True, + extras=frozenset(), # parent did NOT request [dev] + ) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=("pytest ; extra == 'dev'",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + deps = list(provider.get_dependencies(cand)) + # Filter drops the entry -- behaviour identical to the existing + # ``TestGetDependenciesMarkerFilter`` case. introducing_marker + # is irrelevant here because the Requirement is never built. + assert deps == [] + + def test_marker_with_parent_extras_active_carries_marker(self): + """``Requires-Dist: pytest; extra=='dev'`` with parent + ``extras=frozenset({'dev'})`` -> requirement yielded AND + ``introducing_marker`` equals the parsed ``Marker("extra == + 'dev'")``.""" + from pipenv.vendor.packaging.markers import Marker + + cand = _cand( + name="django", + version="4.2.0", + is_wheel=True, + extras=frozenset({"dev"}), # parent DID request [dev] + ) + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=("pytest>=7 ; extra == 'dev'",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + deps = list(provider.get_dependencies(cand)) + # Filter out the synthetic base-version pin emitted because + # parent_extras is non-empty. + non_pin = [d for d in deps if d.name != "django"] + assert len(non_pin) == 1 + dep = non_pin[0] + assert dep.name == "pytest" + assert dep.parent == "django" + assert dep.source == "transitive" + # introducing_marker carries the original parsed marker. + assert dep.introducing_marker is not None + expected = Marker("extra == 'dev'") + assert str(dep.introducing_marker) == str(expected) + + +# ---------------------------------------------------------------------- +# T_M5 (Initiative G Phase 3b): belt-and-braces edge cases for the +# compound-marker shapes that cross the boundary between +# get_dependencies's marker-filter and the introducing_marker +# propagation slot. Coverage on pure_python_provider.py is already at +# 97 %; these tests close the scope by pinning the two tricky cases the +# audit list calls out: a compound platform-AND-python marker (no +# ``extra`` clause) and a compound extra-AND-platform marker (parent +# extras active). +# ---------------------------------------------------------------------- +class TestGetDependenciesT_M5IntroducingMarkerCompound: + """T_M5 — pin that ``get_dependencies`` propagates the FULL parsed + ``Marker`` into ``introducing_marker`` even for compound markers + that combine multiple clauses (``and`` / ``or``). The marker is + NOT split into per-clause Requirements — it round-trips intact so + T_M3's translator can re-emit it as a single ``markers=...`` clause. + """ + + def test_compound_python_and_platform_marker_preserved_intact(self): + """``Requires-Dist: pytest; python_version >= '3.10' and + sys_platform == 'darwin'`` (no ``extra`` clause; parent has no + extras) -> ONE :class:`Requirement` emitted whose + ``introducing_marker`` is the parsed compound :class:`Marker`, + preserving BOTH clauses. + + Pins: + * The compound marker survives the + ``_marker_active_for_extras`` filter under the ``{"extra": + ""}`` overlay (a marker without an ``extra==`` clause is + unaffected by the extras context — that's what makes a plain + ``python_version`` marker on a Requires-Dist line survive). + * The marker is NOT split into per-clause requirements — it's + carried verbatim so T_M3 can emit a single ``markers=...`` + clause on the lockfile entry. + """ + from pipenv.vendor.packaging.markers import Marker + + cand = _cand( + name="django", + version="4.2.0", + is_wheel=True, + extras=frozenset(), # parent has no extras + ) + compound = "python_version >= '3.10' and sys_platform == 'darwin'" + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=(f"pytest ; {compound}",), + ) + # Target_env overlay activates BOTH halves so the filter keeps + # the requirement. python_version >= 3.10 is True for 3.12 and + # sys_platform overlay matches the marker clause exactly. + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={ + "python_version": "3.12", + "sys_platform": "darwin", + }, + ) + + deps = list(provider.get_dependencies(cand)) + assert len(deps) == 1 + dep = deps[0] + assert dep.name == "pytest" + # ``introducing_marker`` carries the FULL compound marker + # verbatim (round-trip through ``Marker.__str__``). + assert dep.introducing_marker is not None + expected = Marker(compound) + assert str(dep.introducing_marker) == str(expected) + # Belt-and-braces: BOTH clauses appear in the marker string + # (a regression that split the marker would emit only one). + marker_str = str(dep.introducing_marker) + assert "python_version" in marker_str + assert "sys_platform" in marker_str + + def test_extra_and_platform_compound_marker_passes_filter_and_propagates( + self, + ): + """``Requires-Dist: pytest; extra == 'dev' and python_version + >= '3.10'`` with parent ``extras=frozenset({'dev'})`` and + target_env Python >= 3.10 -> requirement YIELDED AND + ``introducing_marker`` carries the full compound marker. + + Pins the intersection of the T7 extras-filter and the T_M2 + introducing-marker propagation: + + * The ``extra=='dev'`` half of the marker is satisfied because + parent extras contain ``'dev'`` (``_marker_active_for_extras`` + evaluates the marker under ``{"extra": "dev"}``). + * The ``python_version >= '3.10'`` half evaluates True under + the target_env overlay. + * Both halves AND-True -> the marker is "active" -> requirement + survives the filter. + * Critically, the ENTIRE compound marker (including the + ``extra==`` clause) is preserved on ``introducing_marker`` — + T_M3 will downstream emit the marker verbatim, including the + ``extra==`` clause. This is intentional: the lockfile + consumer (pip install) re-evaluates the marker in the install + environment, where the same ``extra==`` clause filters the + dep at install-time. + """ + from pipenv.vendor.packaging.markers import Marker + + cand = _cand( + name="django", + version="4.2.0", + is_wheel=True, + extras=frozenset({"dev"}), # parent DID request [dev] + ) + compound = "extra == 'dev' and python_version >= '3.10'" + meta = _metadata( + name="django", + version="4.2.0", + requires_dist=(f"pytest>=7 ; {compound}",), + ) + provider = _make_provider( + metadata_fetcher=_metadata_fetcher_stub({cand.url: meta}), + target_env={"python_version": "3.12"}, + ) + + deps = list(provider.get_dependencies(cand)) + # Filter kept the entry (both halves of the AND are True under + # the parent_extras=={'dev'} + python_version=='3.12' overlay). + # Filter out the synthetic base-version pin emitted because + # parent_extras is non-empty. + non_pin = [d for d in deps if d.name != "django"] + assert len(non_pin) == 1 + dep = non_pin[0] + assert dep.name == "pytest" + assert dep.parent == "django" + assert dep.source == "transitive" + # Compound marker preserved end-to-end — including the + # ``extra==`` clause that the filter consumed. ``Marker.__str__`` + # is order-stable for the canonical form so a string round-trip + # compares cleanly. + assert dep.introducing_marker is not None + expected = Marker(compound) + assert str(dep.introducing_marker) == str(expected) + # Belt-and-braces: BOTH clauses appear in the marker. + marker_str = str(dep.introducing_marker) + assert "extra" in marker_str + assert "python_version" in marker_str diff --git a/tests/unit/test_pure_python_provider_smoke.py b/tests/unit/test_pure_python_provider_smoke.py new file mode 100644 index 0000000000..f86d5757ca --- /dev/null +++ b/tests/unit/test_pure_python_provider_smoke.py @@ -0,0 +1,335 @@ +"""Integration smoke tests for :class:`PurePythonProvider` driving +``resolvelib.Resolver`` end-to-end (Initiative G phase 3, T8). + +Scope: this file is the integration-smoke gate for the full +:class:`PurePythonProvider` (T3 through T7). It wires the provider into +``resolvelib.Resolver`` against a synthetic in-memory cache + a +metadata-fetcher stub and asserts on the resulting resolution. + +Two scenarios per the plan T8 validation matrix: + +1. **Happy path** — three-package resolve + (``requests`` + ``certifi`` + ``urllib3``) against a synthetic cache + where each package has 2-3 versions; assert the expected pins + (``2.32.0`` / ``2024.2.2`` / ``2.2.0``) land in the resolved mapping. + +2. **Conflict** — ``a`` requires ``b<2``; ``c`` requires ``b>=2``; + assert :class:`ResolutionImpossible` raises with BOTH causes + (``b<2`` from ``a`` and ``b>=2`` from ``c``) present in the + exception's ``.causes`` list. + +These tests intentionally exercise the helper +:func:`pipenv.resolver.pure_python_provider._drive_resolver` — a +module-level wrapper around ``resolvelib.Resolver`` that T9's +``PurePythonBackend`` will reuse in production. Keeping the wiring +behind one helper means tests and the backend can share the same call +path. + +Critical constraint (matches the module under test): no +``pip._internal.*`` imports in this file. All ``packaging`` / +``resolvelib`` paths route through ``pipenv.vendor.*`` / +``pipenv.patched.pip._vendor.*`` accordingly. +""" +from __future__ import annotations + +from typing import Any + +import pytest + +# --------------------------------------------------------------------------- +# Synthetic infrastructure +# --------------------------------------------------------------------------- + + +_INDEX = "https://pypi.org/simple" + + +class _FakeCache: + """In-memory stand-in for :class:`ParsedManifestCache`. + + Behaves like the real ``cache.get(index_url, package_name)``: + returns ``None`` on miss, a tiny stand-in for + :class:`CachedManifest` (anything exposing ``.candidates``) on hit. + + Mirrors the ``_FakeCache`` in ``test_pure_python_provider.py`` but + is duplicated here rather than imported across test files — the + contract is small and a copy keeps both test modules independent + (T13 may extend the original; T8 should not be perturbed by that). + """ + + class _Manifest: + __slots__ = ("candidates",) + + def __init__(self, candidates: tuple) -> None: + self.candidates = candidates + + def __init__(self, mapping: dict[tuple[str, str], tuple]) -> None: + # mapping: {(index_url, name): (Candidate, ...)} + self._mapping = dict(mapping) + + def get(self, index_url: str, package_name: str): + cands = self._mapping.get((index_url, package_name)) + if cands is None: + return None + return self._Manifest(cands) + + +def _wheel_candidate( + name: str, + version: str, + *, + requires_python: str | None = None, +): + """Build a wheel :class:`Candidate` with inert defaults for the + fields T8 doesn't care about. + + ``name`` is already canonical (lowercase) for every fixture below — + the provider's :meth:`identify` assumes canonical names. + """ + from pipenv.resolver.candidate import Candidate, Hash + + filename = f"{name}-{version}-py3-none-any.whl" + return Candidate( + name=name, + version=version, + url=f"https://pypi.org/simple/{name}/{filename}", + filename=filename, + hashes=frozenset({Hash("sha256", "0" * 64)}), + requires_python=requires_python, + yanked=False, + yanked_reason=None, + upload_time=None, + is_wheel=True, + wheel_tags=None, + extras=frozenset(), + ) + + +def _metadata_fetcher_stub(deps_by_candidate: dict[tuple[str, str], list[str]]): + """Return a ``metadata_fetcher`` callable suitable for the provider. + + ``deps_by_candidate`` is ``{(name, version): [raw_requires_dist, ...]}``. + The callable matches the shape the provider expects: + ``metadata_fetcher(candidate) -> CoreMetadata``. + + A candidate not present in the mapping is treated as "no deps" + (rather than asserting) — that's the leaf-package case + (``certifi`` and ``urllib3`` in the happy-path scenario). + """ + from pipenv.resolver.pure_python_metadata import CoreMetadata + + def fetch(candidate): + raw = deps_by_candidate.get( + (candidate.name, candidate.version), [] + ) + return CoreMetadata( + name=candidate.name, + version=candidate.version, + requires_python=candidate.requires_python or "", + requires_dist=tuple(raw), + provides_extras=frozenset(), + summary=None, + ) + + return fetch + + +def _make_provider( + *, + cache: _FakeCache, + metadata_fetcher: Any, + target_env: dict | None = None, +): + from pipenv.resolver.pure_python_provider import PurePythonProvider + + return PurePythonProvider( + cache=cache, + fetcher=object(), # T8 fixtures never need lazy population + metadata_fetcher=metadata_fetcher, + target_env=target_env or {"python_version": "3.12"}, + index_urls=[_INDEX], + allow_prereleases=False, + ) + + +# --------------------------------------------------------------------------- +# T8 — Happy path: 3-package resolve +# --------------------------------------------------------------------------- + + +class TestDriveResolverHappyPath: + """Plan T8 bullet 1: resolve ``requests`` + ``certifi`` + ``urllib3`` + against a synthetic 3-package cache; expected pins land. + + Cache layout (per plan T8): + + * ``requests``: 2.30.0, 2.31.0, 2.32.0; ``2.32.0`` requires + ``certifi>=2017.4.17`` and ``urllib3>=1.21.1,<3``. + * ``certifi``: 2024.1.1, 2024.2.2; no deps. + * ``urllib3``: 1.26.18, 2.0.7, 2.2.0; no deps. + + Top-level requirement: ``requests`` (``"*"``). + + Expected resolved pins: + + * ``requests==2.32.0`` (latest of the three versions), + * ``certifi==2024.2.2`` (latest, satisfies ``>=2017.4.17``), + * ``urllib3==2.2.0`` (latest, satisfies ``>=1.21.1,<3``). + """ + + def _build_fixture(self): + requests_versions = [ + _wheel_candidate("requests", "2.30.0"), + _wheel_candidate("requests", "2.31.0"), + _wheel_candidate("requests", "2.32.0"), + ] + certifi_versions = [ + _wheel_candidate("certifi", "2024.1.1"), + _wheel_candidate("certifi", "2024.2.2"), + ] + urllib3_versions = [ + _wheel_candidate("urllib3", "1.26.18"), + _wheel_candidate("urllib3", "2.0.7"), + _wheel_candidate("urllib3", "2.2.0"), + ] + + cache = _FakeCache( + { + (_INDEX, "requests"): tuple(requests_versions), + (_INDEX, "certifi"): tuple(certifi_versions), + (_INDEX, "urllib3"): tuple(urllib3_versions), + } + ) + + # Only ``requests 2.32.0`` has deps in this fixture. The + # provider only fetches metadata for the candidate it actually + # picks (resolvelib's lazy expansion), so we don't bother + # seeding entries for the older requests versions. + deps_by_candidate = { + ("requests", "2.32.0"): [ + "certifi>=2017.4.17", + "urllib3>=1.21.1,<3", + ], + } + fetcher = _metadata_fetcher_stub(deps_by_candidate) + return cache, fetcher + + def test_happy_path_resolves_expected_pins(self): + from pipenv.resolver.pure_python_provider import _drive_resolver + from pipenv.resolver.pure_python_requirement import Requirement + + cache, fetcher = self._build_fixture() + provider = _make_provider(cache=cache, metadata_fetcher=fetcher) + + top_level = [Requirement.from_pipfile_entry("requests", "*")] + + result = _drive_resolver(top_level, provider) + + # ``result.mapping`` is keyed on the provider's identifier + # tuple — ``(canonical_name, frozenset(extras))``. + mapping = result.mapping + pins = { + name: cand.version + for (name, _extras), cand in mapping.items() + } + + # Spot-check each expected pin one at a time so a regression + # surfaces the exact axis (which version got picked) rather + # than a single opaque equality failure. + assert pins.get("requests") == "2.32.0" + assert pins.get("certifi") == "2024.2.2" + assert pins.get("urllib3") == "2.2.0" + + # And the closed-world check: no surprise extra packages + # leaked into the resolution. Three keys, three packages. + assert set(pins.keys()) == {"requests", "certifi", "urllib3"} + + +# --------------------------------------------------------------------------- +# T8 — Conflict scenario +# --------------------------------------------------------------------------- + + +class TestDriveResolverConflict: + """Plan T8 bullet 2: conflicting top-level requirements raise + :class:`ResolutionImpossible` with both causes present. + + Cache layout (per plan T8): + + * ``a 1.0.0``: requires ``b<2``. + * ``c 1.0.0``: requires ``b>=2``. + * ``b``: 1.5.0, 2.0.0, 2.1.0; no deps. + + Top-level reqs: ``a`` (``"*"``) and ``c`` (``"*"``). No version + of ``b`` satisfies both ``<2`` and ``>=2`` simultaneously, so + resolvelib raises ``ResolutionImpossible``. + """ + + def _build_fixture(self): + a_versions = [_wheel_candidate("a", "1.0.0")] + c_versions = [_wheel_candidate("c", "1.0.0")] + b_versions = [ + _wheel_candidate("b", "1.5.0"), + _wheel_candidate("b", "2.0.0"), + _wheel_candidate("b", "2.1.0"), + ] + cache = _FakeCache( + { + (_INDEX, "a"): tuple(a_versions), + (_INDEX, "c"): tuple(c_versions), + (_INDEX, "b"): tuple(b_versions), + } + ) + deps_by_candidate = { + ("a", "1.0.0"): ["b<2"], + ("c", "1.0.0"): ["b>=2"], + } + fetcher = _metadata_fetcher_stub(deps_by_candidate) + return cache, fetcher + + def test_conflict_raises_resolution_impossible_with_both_causes(self): + from pipenv.patched.pip._vendor.resolvelib import ResolutionImpossible + + from pipenv.resolver.pure_python_provider import _drive_resolver + from pipenv.resolver.pure_python_requirement import Requirement + + cache, fetcher = self._build_fixture() + provider = _make_provider(cache=cache, metadata_fetcher=fetcher) + + top_level = [ + Requirement.from_pipfile_entry("a", "*"), + Requirement.from_pipfile_entry("c", "*"), + ] + + with pytest.raises(ResolutionImpossible) as excinfo: + _drive_resolver(top_level, provider) + + # ``.causes`` is a list of ``RequirementInformation`` + # namedtuples — ``(requirement, parent)``. Both transitive + # requirements (``b<2`` from ``a`` and ``b>=2`` from ``c``) + # must appear; assert via the rendered specifier strings so + # the test is robust to ordering changes in ``.causes``. + causes = list(excinfo.value.causes) + # All causes should be about ``b`` — that's the package whose + # constraints conflict. This is also a useful sanity check + # that resolvelib reported on the right axis. + cause_names = {row.requirement.name for row in causes} + assert cause_names == {"b"} + + # Each cause carries a ``SpecifierSet``; render to a sorted + # list-of-strings to compare. ``"<2"`` lives in one cause, + # ``">=2"`` in the other — both must be present. + cause_specs = sorted( + str(row.requirement.specifier) for row in causes + ) + assert "<2" in cause_specs + assert ">=2" in cause_specs + + # Bonus: each cause records the parent that produced it + # (transitive requirements carry ``parent``). Pin both + # parent identities so a future refactor of + # ``get_dependencies`` doesn't silently drop the attribution. + parents = { + row.requirement.parent for row in causes if row.requirement.parent + } + assert parents == {"a", "c"} diff --git a/tests/unit/test_pure_python_requirement.py b/tests/unit/test_pure_python_requirement.py new file mode 100644 index 0000000000..c298b17886 --- /dev/null +++ b/tests/unit/test_pure_python_requirement.py @@ -0,0 +1,688 @@ +"""Unit tests for :mod:`pipenv.resolver.pure_python_requirement` +(Initiative G Phase 3, T1). + +This file is the RED-phase test suite that pins T1's contract. T11 +extends this file later with the broader coverage matrix; the minimum +acceptance gate from the plan brief is: + +* Round-trip construction with every field set; per-field accessors + return what we passed in. +* Equality + hashability — instances must be usable inside a + ``frozenset``. +* :meth:`Requirement.from_pipfile_entry` handles the three canonical + Pipfile shapes: + + - ``"*"`` → empty ``SpecifierSet``. + - ``">=4.0,<6"`` → parsed ``SpecifierSet``. + - dict ``{"version": ">=4.0", "extras": ["argon2"]}`` → extras + populated, version parsed. + +* Name canonicalisation: ``Django_Rest`` → ``django-rest``. + +Everything else (negative paths, marker handling on dict form, the +``constraint`` source label, etc.) is covered by T11. +""" + +from __future__ import annotations + +import pytest + +from pipenv.resolver.pure_python_requirement import Requirement +from pipenv.vendor.packaging.markers import Marker +from pipenv.vendor.packaging.specifiers import SpecifierSet + +# --------------------------------------------------------------------------- +# Round-trip + per-field accessors +# --------------------------------------------------------------------------- + + +def test_requirement_round_trip_all_fields() -> None: + """A Requirement built with every field set returns every field set.""" + spec = SpecifierSet(">=4.0,<6") + marker = Marker("python_version >= '3.10'") + req = Requirement( + name="django", + specifier=spec, + extras=frozenset({"argon2"}), + marker=marker, + source="pipfile", + parent=None, + ) + assert req.name == "django" + assert req.specifier == spec + assert req.extras == frozenset({"argon2"}) + assert req.marker == marker + assert req.source == "pipfile" + assert req.parent is None + + +def test_requirement_transitive_with_parent() -> None: + """Transitive requirements carry the parent candidate name.""" + req = Requirement( + name="urllib3", + specifier=SpecifierSet(">=1.26"), + extras=frozenset(), + marker=None, + source="transitive", + parent="requests", + ) + assert req.source == "transitive" + assert req.parent == "requests" + + +# --------------------------------------------------------------------------- +# Equality + hashability (frozenset membership) +# --------------------------------------------------------------------------- + + +def test_requirement_equality_same_fields() -> None: + """Two Requirements with identical fields compare equal and hash equal.""" + a = Requirement( + name="django", + specifier=SpecifierSet(">=4.0"), + extras=frozenset({"argon2"}), + marker=None, + source="pipfile", + parent=None, + ) + b = Requirement( + name="django", + specifier=SpecifierSet(">=4.0"), + extras=frozenset({"argon2"}), + marker=None, + source="pipfile", + parent=None, + ) + assert a == b + assert hash(a) == hash(b) + + +def test_requirement_inequality_on_extras() -> None: + """Different extras → unequal requirements.""" + a = Requirement( + name="django", + specifier=SpecifierSet(">=4.0"), + extras=frozenset({"argon2"}), + marker=None, + source="pipfile", + parent=None, + ) + b = Requirement( + name="django", + specifier=SpecifierSet(">=4.0"), + extras=frozenset(), + marker=None, + source="pipfile", + parent=None, + ) + assert a != b + + +def test_requirement_is_frozenset_member() -> None: + """Requirement must be usable inside a ``frozenset`` (resolvelib needs this).""" + a = Requirement( + name="django", + specifier=SpecifierSet(">=4.0"), + extras=frozenset(), + marker=None, + source="pipfile", + parent=None, + ) + b = Requirement( + name="django", + specifier=SpecifierSet(">=4.0"), + extras=frozenset(), + marker=None, + source="pipfile", + parent=None, + ) + bag = frozenset({a, b}) + # Equal objects collapse to a single member. + assert len(bag) == 1 + assert a in bag + + +def test_requirement_is_frozen() -> None: + """Frozen dataclass refuses attribute assignment.""" + req = Requirement( + name="django", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=None, + source="pipfile", + parent=None, + ) + with pytest.raises((AttributeError, Exception)): + req.name = "flask" # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# from_pipfile_entry shapes +# --------------------------------------------------------------------------- + + +def test_from_pipfile_entry_star_is_empty_specifier() -> None: + """``"*"`` → empty ``SpecifierSet`` (any version is acceptable).""" + req = Requirement.from_pipfile_entry("django", "*") + assert req.name == "django" + assert req.specifier == SpecifierSet("") + assert len(list(req.specifier)) == 0 + assert req.extras == frozenset() + assert req.marker is None + assert req.source == "pipfile" + + +def test_from_pipfile_entry_string_specifier() -> None: + """A bare string is parsed as a specifier.""" + req = Requirement.from_pipfile_entry("django", ">=4.0,<6") + assert req.name == "django" + assert req.specifier == SpecifierSet(">=4.0,<6") + assert req.extras == frozenset() + + +def test_from_pipfile_entry_dict_with_extras() -> None: + """Dict form populates version + extras.""" + req = Requirement.from_pipfile_entry( + "django", {"version": ">=4.0", "extras": ["argon2"]} + ) + assert req.name == "django" + assert req.specifier == SpecifierSet(">=4.0") + assert req.extras == frozenset({"argon2"}) + + +def test_from_pipfile_entry_dict_star_version() -> None: + """Dict form with ``"version": "*"`` → empty SpecifierSet.""" + req = Requirement.from_pipfile_entry("django", {"version": "*"}) + assert req.specifier == SpecifierSet("") + + +def test_from_pipfile_entry_dict_with_markers() -> None: + """Dict form parses a markers string into a Marker.""" + req = Requirement.from_pipfile_entry( + "django", + {"version": ">=4.0", "markers": "python_version >= '3.10'"}, + ) + assert req.marker is not None + assert str(req.marker) == str(Marker("python_version >= '3.10'")) + + +def test_from_pipfile_entry_canonicalizes_name() -> None: + """``Django_Rest`` → ``django-rest`` (PEP 503).""" + req = Requirement.from_pipfile_entry("Django_Rest", "*") + assert req.name == "django-rest" + + +def test_from_pipfile_entry_source_default_is_pipfile() -> None: + """Default ``source`` for a Pipfile entry is ``"pipfile"``.""" + req = Requirement.from_pipfile_entry("django", "*") + assert req.source == "pipfile" + + +def test_from_pipfile_entry_source_override() -> None: + """Caller may override ``source`` (e.g. for constraint entries).""" + req = Requirement.from_pipfile_entry( + "django", ">=4.0", source="constraint" + ) + assert req.source == "constraint" + + +def test_from_pipfile_entry_parent_propagates() -> None: + """``parent`` keyword propagates onto the resulting Requirement.""" + req = Requirement.from_pipfile_entry( + "urllib3", + ">=1.26", + source="transitive", + parent="requests", + ) + assert req.parent == "requests" + assert req.source == "transitive" + + +# --------------------------------------------------------------------------- +# T11 extension — coverage matrix +# --------------------------------------------------------------------------- +# +# The 15 tests above (added under T1) hit 97 % of +# ``pipenv/resolver/pure_python_requirement.py`` on their own; this +# section pins the remaining branch (the unsupported-shape ``TypeError``) +# and the edge cases enumerated in initiative-g-phase3-plan.md T11: +# dict shapes that include keys the dataclass deliberately ignores +# (editable / VCS), empty + whitespace version strings, marker / +# extras / version mixed, the canonicalisation corner-cases, the +# ``source`` Literal escape hatch. + + +def test_from_pipfile_entry_rejects_unsupported_value_type() -> None: + """Non-str / non-dict values raise ``TypeError`` with a clear message. + + Covers the ``else`` branch at the bottom of ``from_pipfile_entry`` + (line 174 of pure_python_requirement.py) — the loud-failure rail + for upstream-parser bugs. + """ + with pytest.raises(TypeError) as exc_info: + Requirement.from_pipfile_entry("django", 42) # type: ignore[arg-type] + msg = str(exc_info.value) + assert "django" in msg + assert "int" in msg + + +def test_from_pipfile_entry_rejects_list_value() -> None: + """A list is also rejected (only str / dict are valid).""" + with pytest.raises(TypeError) as exc_info: + Requirement.from_pipfile_entry( + "django", [">=4.0"] # type: ignore[arg-type] + ) + assert "list" in str(exc_info.value) + + +def test_from_pipfile_entry_rejects_none_value() -> None: + """``None`` is not a valid Pipfile entry shape — raise TypeError.""" + with pytest.raises(TypeError): + Requirement.from_pipfile_entry( + "django", None # type: ignore[arg-type] + ) + + +# --------------------------------------------------------------------------- +# Dict-form variants — keys the constraint node deliberately ignores +# --------------------------------------------------------------------------- + + +def test_from_pipfile_entry_dict_with_editable_key_ignored() -> None: + """``editable=True`` on the Pipfile dict is silently ignored. + + T11 audit: ``from_pipfile_entry`` builds the *constraint* node only — + it consumes ``version`` / ``extras`` / ``markers`` and leaves + artefact-shape keys (``editable``, ``path``, ``git``, ``ref``, + ``index``, ...) for the Candidate / VCS pipeline. The pure-Python + backend doesn't support VCS / editable sources in Phase 3, but + rejecting them at the constraint-construction layer would be + premature: Pipfile parsing upstream is expected to route those + entries away from the pure-python provider. We pin the current + "silently ignore" behaviour so a future hardening pass is explicit. + """ + req = Requirement.from_pipfile_entry( + "django", {"version": "*", "editable": True} + ) + assert req.name == "django" + assert req.specifier == SpecifierSet("") + assert req.extras == frozenset() + assert req.marker is None + + +def test_from_pipfile_entry_dict_with_vcs_keys_ignored() -> None: + """``git`` / ``ref`` keys on the dict are ignored — see audit note above.""" + req = Requirement.from_pipfile_entry( + "django", + { + "git": "https://github.com/django/django.git", + "ref": "main", + "version": "*", + }, + ) + # The constraint emerges with an empty SpecifierSet — VCS routing + # is the caller's problem, not Requirement's. + assert req.specifier == SpecifierSet("") + + +def test_from_pipfile_entry_dict_with_index_and_path_keys_ignored() -> None: + """``index`` / ``path`` keys on the dict are ignored — same rationale.""" + req = Requirement.from_pipfile_entry( + "django", + {"version": ">=4.0", "index": "pypi", "path": "./local"}, + ) + assert req.specifier == SpecifierSet(">=4.0") + + +# --------------------------------------------------------------------------- +# Specifier corner cases +# --------------------------------------------------------------------------- + + +def test_from_pipfile_entry_dict_without_version_key() -> None: + """A dict missing ``version`` falls through to ``"*"`` (empty specifier).""" + req = Requirement.from_pipfile_entry("django", {"extras": ["argon2"]}) + assert req.specifier == SpecifierSet("") + assert req.extras == frozenset({"argon2"}) + + +def test_from_pipfile_entry_dict_with_empty_version_string() -> None: + """``"version": ""`` is treated the same as ``"*"`` — empty SpecifierSet. + + Covers the ``spec_string in (None, "", "*")`` branch when the + dict supplies an explicit empty string. + """ + req = Requirement.from_pipfile_entry("django", {"version": ""}) + assert req.specifier == SpecifierSet("") + + +def test_from_pipfile_entry_bare_empty_string_is_empty_specifier() -> None: + """A bare ``""`` value also flattens to an empty SpecifierSet (no error).""" + req = Requirement.from_pipfile_entry("django", "") + assert req.specifier == SpecifierSet("") + + +def test_from_pipfile_entry_string_with_whitespace_normalised() -> None: + """SpecifierSet strips surrounding whitespace — round-trip equality holds.""" + req = Requirement.from_pipfile_entry("django", " >=4.0,<6 ") + assert req.specifier == SpecifierSet(">=4.0,<6") + + +# --------------------------------------------------------------------------- +# Combined extras + markers + version +# --------------------------------------------------------------------------- + + +def test_from_pipfile_entry_dict_with_version_extras_and_markers() -> None: + """Dict with all three Pipfile-relevant keys populates all three fields.""" + req = Requirement.from_pipfile_entry( + "django", + { + "version": ">=4.0", + "extras": ["argon2", "bcrypt"], + "markers": "python_version >= '3.10'", + }, + ) + assert req.specifier == SpecifierSet(">=4.0") + assert req.extras == frozenset({"argon2", "bcrypt"}) + assert req.marker is not None + assert str(req.marker) == str(Marker("python_version >= '3.10'")) + + +def test_from_pipfile_entry_dict_with_extras_none_treated_as_empty() -> None: + """``"extras": None`` flattens to an empty frozenset (``or ()`` guard).""" + req = Requirement.from_pipfile_entry( + "django", {"version": "*", "extras": None} + ) + assert req.extras == frozenset() + + +def test_from_pipfile_entry_dict_with_markers_none() -> None: + """``"markers": None`` results in ``marker is None`` (falsy branch).""" + req = Requirement.from_pipfile_entry( + "django", {"version": "*", "markers": None} + ) + assert req.marker is None + + +def test_from_pipfile_entry_dict_with_empty_markers_string() -> None: + """``"markers": ""`` is falsy → ``marker is None``.""" + req = Requirement.from_pipfile_entry( + "django", {"version": "*", "markers": ""} + ) + assert req.marker is None + + +def test_from_pipfile_entry_extras_iterable_coerces_to_strings() -> None: + """Each extra is run through ``str()`` — non-string iterables coerce.""" + # The dataclass field is ``frozenset[str]``; the loud-failure + # contract is that *some* iterable yields *some* values that + # ``str(...)`` accepts. This pins the coercion. + req = Requirement.from_pipfile_entry( + "django", {"version": "*", "extras": ("argon2",)} + ) + assert req.extras == frozenset({"argon2"}) + + +# --------------------------------------------------------------------------- +# Name canonicalisation corner cases +# --------------------------------------------------------------------------- + + +def test_from_pipfile_entry_canonicalises_dots_and_underscores() -> None: + """``"Foo.Bar_Baz"`` → ``"foo-bar-baz"`` (PEP 503 corner case).""" + req = Requirement.from_pipfile_entry("Foo.Bar_Baz", "*") + assert req.name == "foo-bar-baz" + + +def test_from_pipfile_entry_canonicalises_mixed_separator_runs() -> None: + """Mixed runs of ``_`` / ``-`` / ``.`` collapse per PEP 503.""" + req = Requirement.from_pipfile_entry("a__b--c..d", "*") + assert req.name == "a-b-c-d" + + +def test_from_pipfile_entry_canonicalises_numeric_prefix() -> None: + """``"3M"`` → ``"3m"`` — numeric-prefixed names are valid and lowercased.""" + req = Requirement.from_pipfile_entry("3M", "*") + assert req.name == "3m" + + +# --------------------------------------------------------------------------- +# ``source`` Literal — runtime semantics +# --------------------------------------------------------------------------- + + +def test_requirement_source_literal_not_enforced_at_runtime() -> None: + """``source`` is a ``typing.Literal`` — static checkers reject bad values + but the runtime does not. + + T11 audit: the dataclass deliberately omits a ``__post_init__`` + runtime check on ``source``. The rationale (per + pure_python_requirement.py line 45–49): pyright / mypy catch + typos at authoring time; spending CPU on a runtime branch check + inside the resolver's hot loop is wasteful. We pin the + "no runtime guard" behaviour so a future contributor who adds + validation has to update this test deliberately. + """ + req = Requirement( + name="django", + specifier=SpecifierSet(""), + extras=frozenset(), + marker=None, + source="not-a-valid-value", # type: ignore[arg-type] + parent=None, + ) + # No exception raised; the value flows through verbatim. + assert req.source == "not-a-valid-value" + + +def test_from_pipfile_entry_source_literal_passthrough() -> None: + """``from_pipfile_entry`` does not validate ``source`` either — + matches the dataclass-level audit above.""" + req = Requirement.from_pipfile_entry( + "django", "*", source="bogus" # type: ignore[arg-type] + ) + assert req.source == "bogus" + + +# --------------------------------------------------------------------------- +# T_M1 — ``introducing_marker`` field (Initiative G Phase 3b) +# --------------------------------------------------------------------------- +# +# T_M1 adds an optional ``introducing_marker: Marker | None = None`` slot +# to :class:`Requirement` so T_M2 can populate it from each +# ``Requires-Dist`` line (e.g. ``pytest; python_version < '3.10'``) and +# T_M3 can read it in ``_translate_mapping`` when emitting markers on +# the lockfile entry. ``from_pipfile_entry`` is *not* widened — the +# helper is only called for top-level Pipfile entries (which never +# carry an introducing marker by definition); transitives are built via +# direct ``Requirement(...)`` construction in +# :meth:`PurePythonProvider.get_dependencies`. + + +class TestRequirementIntroducingMarker: + """Pin the T_M1 contract for ``Requirement.introducing_marker``.""" + + def test_introducing_marker_defaults_to_none(self) -> None: + """Pipfile-shape construction (the only path through + :meth:`Requirement.from_pipfile_entry`) leaves + ``introducing_marker`` as ``None`` — the helper is not widened + and the field defaults to ``None`` on the dataclass.""" + req = Requirement.from_pipfile_entry("django", "*") + assert req.introducing_marker is None + + def test_introducing_marker_field_round_trip(self) -> None: + """Direct construction with an ``introducing_marker`` round-trips + the same :class:`Marker` instance back out.""" + m = Marker("python_version < '3.10'") + req = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=m, + ) + assert req.introducing_marker is m + + def test_introducing_marker_defaults_to_none_on_direct_construction( + self, + ) -> None: + """Existing call sites that build :class:`Requirement` without + the new kwarg get ``introducing_marker=None`` for free — proves + the field's default and preserves backwards-compatible + kwarg-style construction.""" + req = Requirement( + name="urllib3", + specifier=SpecifierSet(">=1.26"), + extras=frozenset(), + marker=None, + source="transitive", + parent="requests", + ) + assert req.introducing_marker is None + + def test_introducing_marker_equality_same_marker(self) -> None: + """Two Requirements with the same ``introducing_marker`` value + (constructed independently from equal marker strings) compare + equal.""" + a = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=Marker("python_version < '3.10'"), + ) + b = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=Marker("python_version < '3.10'"), + ) + assert a == b + assert hash(a) == hash(b) + + def test_introducing_marker_inequality_different_marker(self) -> None: + """Different ``introducing_marker`` values produce unequal + Requirements — T_M3 relies on this when collapsing multiple + introductions of the same transitive.""" + a = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=Marker("python_version < '3.10'"), + ) + b = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=Marker("python_version >= '3.10'"), + ) + assert a != b + + def test_introducing_marker_inequality_none_vs_marker(self) -> None: + """``introducing_marker=None`` vs a populated marker is an + inequality — pins that the field participates in dataclass + equality.""" + a = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=None, + ) + b = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=Marker("python_version < '3.10'"), + ) + assert a != b + + def test_introducing_marker_frozenset_membership(self) -> None: + """Requirements carrying an ``introducing_marker`` are still + hashable — :class:`Marker` is hashable in + ``pipenv.vendor.packaging`` (markers.py:329) so the frozen + dataclass's derived ``__hash__`` works without an override.""" + m = Marker("python_version < '3.10'") + a = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=m, + ) + b = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=Marker("python_version < '3.10'"), + ) + bag = frozenset({a, b}) + # Equal Requirements (including their introducing markers) + # collapse to a single member. + assert len(bag) == 1 + assert a in bag + + def test_introducing_marker_dataclass_replace_preserves_field( + self, + ) -> None: + """``dataclasses.replace`` round-trips ``introducing_marker`` + — T_M2 uses this pattern when rebuilding a marker-aware variant + of a transitive Requirement.""" + import dataclasses as _dc + + original = Requirement( + name="pytest", + specifier=SpecifierSet(">=7"), + extras=frozenset(), + marker=None, + source="transitive", + parent="django", + introducing_marker=None, + ) + m = Marker("python_version < '3.10'") + updated = _dc.replace(original, introducing_marker=m) + assert updated.introducing_marker is m + # The replace produced a *new* instance; the original stays + # unmodified (frozen-dataclass guarantee). + assert original.introducing_marker is None + + def test_from_pipfile_entry_does_not_accept_introducing_marker_kwarg( + self, + ) -> None: + """``from_pipfile_entry`` is deliberately NOT widened with an + ``introducing_marker`` kwarg — the helper only builds top-level + Pipfile constraints, never transitives. Calling it with the + kwarg raises :class:`TypeError` from Python's + keyword-argument-handling, pinning the design decision.""" + with pytest.raises(TypeError): + Requirement.from_pipfile_entry( + "django", + "*", + introducing_marker=Marker( # type: ignore[call-arg] + "python_version < '3.10'" + ), + ) diff --git a/tests/unit/test_pure_python_sdist.py b/tests/unit/test_pure_python_sdist.py new file mode 100644 index 0000000000..591687751c --- /dev/null +++ b/tests/unit/test_pure_python_sdist.py @@ -0,0 +1,897 @@ +"""Unit tests for :mod:`pipenv.resolver.pure_python_sdist` +(Initiative G Phase 3b, T_S1). + +Coverage matrix from the plan brief: + +1. Happy path with a hand-built synthetic sdist + setuptools legacy backend. +2. Cache round-trip (second call hits cache, not HTTP). +3. HTTP failure → ``SdistBuildError`` with ``download`` in the message. +4. Corrupt archive bytes → ``SdistBuildError`` with ``corrupt``. +5. Pyproject pointing at a non-existent build backend → ``SdistBuildError`` + with ``build backend failed``. +6. Sdist without pyproject.toml → legacy fallback path succeeds. +7. Path-traversal protection on tar member names. + +The "happy path" tests really invoke setuptools, so they're a touch +slower than the wheel-METADATA unit tests; the repo has no ``slow`` +marker convention so we accept the ~5 s runtime. + +The session is a duck-typed :class:`unittest.mock.MagicMock` matching +the urllib3 shape :mod:`pipenv.resolver.pure_python_metadata` uses +(``.status`` / ``.data`` / ``.headers``). +""" + +from __future__ import annotations + +import io +import tarfile +import zipfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +# --------------------------------------------------------------------------- +# Test infrastructure: skip the PEP 517 isolated venv +# --------------------------------------------------------------------------- +# Production builds sdists in a throwaway venv via :mod:`build`'s +# ``DefaultIsolatedEnv`` so package-specific build backends +# (poetry-core, hatchling, …) can be installed. Spinning up that +# venv per-test would add multi-second overhead and pull a real +# network of build-requires into unit tests. Replace the helper +# with a pyproject_hooks call that runs in the test process's own +# Python — same surface, microseconds instead of seconds, and the +# happy-path fixtures only need ``setuptools`` which the test env +# already carries. +# --------------------------------------------------------------------------- + + +def _no_isolation_build(source_root: Path, metadata_dir: Path) -> Path: + from pipenv.patched.pip._vendor.pyproject_hooks import ( + BuildBackendHookCaller, + ) + from pipenv.resolver.pure_python_sdist import _resolve_build_backend + + backend, backend_path = _resolve_build_backend(source_root) + caller = BuildBackendHookCaller( + source_dir=str(source_root), + build_backend=backend, + backend_path=backend_path, + ) + relative = caller.prepare_metadata_for_build_wheel(str(metadata_dir)) + return Path(metadata_dir) / relative + + +@pytest.fixture(autouse=True) +def _patch_isolated_build(monkeypatch): + """Default test-wide override of the isolated-build helper. + + Tests that need to inject specific failure modes (timeout, + missing METADATA, non-UTF-8 METADATA, raised exception) override + this fixture's effect with their own ``monkeypatch.setattr``. + """ + from pipenv.resolver import pure_python_sdist + + monkeypatch.setattr( + pure_python_sdist, "_build_metadata_in_isolated_env", _no_isolation_build + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_response(*, status: int = 200, data: bytes = b"") -> MagicMock: + """Build a urllib3-response-shaped mock.""" + response = MagicMock() + response.status = status + response.data = data + response.headers = {} + return response + + +def _make_session(response_bytes: bytes, *, status: int = 200) -> MagicMock: + """Session whose ``request("GET", url, ...)`` returns ``response_bytes``.""" + session = MagicMock() + session.request = MagicMock( + return_value=_make_response(status=status, data=response_bytes) + ) + return session + + +def _make_candidate(filename: str, *, url: str | None = None) -> SimpleNamespace: + """Minimal candidate-shaped object — ``.url`` + ``.filename`` only.""" + actual_url = url or f"https://example.test/sdists/{filename}" + return SimpleNamespace(url=actual_url, filename=filename) + + +def _build_sdist_tarball( + *, + pkg_name: str, + version: str, + pyproject_toml: bytes | None, + extra_files: dict[str, bytes] | None = None, +) -> bytes: + """Build a tar.gz sdist in memory. + + The archive contains a single top-level ``{pkg_name}-{version}/`` + directory. When ``pyproject_toml`` is ``None`` the + ``pyproject.toml`` is omitted (forcing the legacy fallback). + """ + top = f"{pkg_name}-{version}" + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tf: + def add(name: str, payload: bytes) -> None: + info = tarfile.TarInfo(name=f"{top}/{name}") + info.size = len(payload) + tf.addfile(info, io.BytesIO(payload)) + + if pyproject_toml is not None: + add("pyproject.toml", pyproject_toml) + + # PKG-INFO is what setuptools' legacy backend reads to extract + # name/version when no setup.py is invoked; including it makes + # the synthetic sdist a valid PEP 643 sdist. + add( + "PKG-INFO", + ( + f"Metadata-Version: 2.1\n" + f"Name: {pkg_name}\n" + f"Version: {version}\n" + f"Summary: synthetic test sdist\n" + ).encode(), + ) + + # A setup.py is required by the legacy setuptools backend; for + # PEP 517 backends that aren't legacy, callers can pass a + # pyproject and skip setup.py via extra_files. + if extra_files is None or "setup.py" not in extra_files: + add( + "setup.py", + ( + "from setuptools import setup\n" + f"setup(name={pkg_name!r}, version={version!r}, " + f"description='synthetic test sdist')\n" + ).encode(), + ) + + # Source dir so the build doesn't produce an empty wheel. + add(f"{pkg_name.replace('-', '_')}/__init__.py", b"") + + for name, payload in (extra_files or {}).items(): + add(name, payload) + + return buf.getvalue() + + +def _build_traversal_tarball() -> bytes: + """Build a tarball containing a ``../etc/passwd`` member.""" + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tf: + # First a normal directory so the archive looks plausible. + info = tarfile.TarInfo(name="malicious-1.0/") + info.type = tarfile.DIRTYPE + tf.addfile(info) + # Then the bad member. + info = tarfile.TarInfo(name="../etc/passwd") + payload = b"pwned" + info.size = len(payload) + tf.addfile(info, io.BytesIO(payload)) + return buf.getvalue() + + +# --------------------------------------------------------------------------- +# 1. Happy path — pyproject + setuptools.build_meta:__legacy__ +# --------------------------------------------------------------------------- + + +class TestHappyPath: + def test_extracts_metadata_via_setuptools_legacy_backend(self): + from pipenv.resolver.pure_python_sdist import ( + extract_metadata_from_sdist, + ) + + # No pyproject — exercises the legacy fallback inside + # _resolve_build_backend AND the legacy setuptools backend + # itself (the most common shape for old PyPI sdists). + sdist = _build_sdist_tarball( + pkg_name="mypkg", + version="1.0", + pyproject_toml=None, + ) + session = _make_session(sdist) + candidate = _make_candidate("mypkg-1.0.tar.gz") + + result = extract_metadata_from_sdist(candidate, session) + + assert result.name == "mypkg" + assert result.version == "1.0" + # Build-backend produces a real METADATA so Summary survives. + assert result.summary == "synthetic test sdist" + + def test_extracts_metadata_from_zip_sdist(self): + from pipenv.resolver.pure_python_sdist import ( + extract_metadata_from_sdist, + ) + + # Build a zip-format sdist (some legacy projects publish them). + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr( + "zippkg-2.0/PKG-INFO", + "Metadata-Version: 2.1\nName: zippkg\nVersion: 2.0\n", + ) + zf.writestr( + "zippkg-2.0/setup.py", + "from setuptools import setup\nsetup(name='zippkg', version='2.0')\n", + ) + zf.writestr("zippkg-2.0/zippkg/__init__.py", "") + sdist = buf.getvalue() + session = _make_session(sdist) + candidate = _make_candidate("zippkg-2.0.zip") + + result = extract_metadata_from_sdist(candidate, session) + + assert result.name == "zippkg" + assert result.version == "2.0" + + +# --------------------------------------------------------------------------- +# 2. Cache round-trip +# --------------------------------------------------------------------------- + + +class TestCacheRoundTrip: + def test_second_call_hits_cache_not_http(self, tmp_path: Path): + from pipenv.resolver.pure_python_metadata import MetadataCache + from pipenv.resolver.pure_python_sdist import ( + extract_metadata_from_sdist, + ) + + sdist = _build_sdist_tarball( + pkg_name="cachekit", + version="0.1", + pyproject_toml=None, + ) + candidate = _make_candidate("cachekit-0.1.tar.gz") + cache = MetadataCache(tmp_path / "metacache") + + # First call: real session + real build. + session1 = _make_session(sdist) + first = extract_metadata_from_sdist(candidate, session1, cache=cache) + assert first.name == "cachekit" + assert session1.request.call_count == 1 + + # Second call: cache hit, the HTTP session should NEVER be + # touched. We pass a mock whose .request raises so any + # accidental call would fail loudly. + session2 = MagicMock() + session2.request.side_effect = AssertionError("HTTP must not be called") + second = extract_metadata_from_sdist(candidate, session2, cache=cache) + assert second.name == "cachekit" + assert second.version == "0.1" + assert session2.request.call_count == 0 + + +# --------------------------------------------------------------------------- +# 3. HTTP failure +# --------------------------------------------------------------------------- + + +class TestHttpFailure: + def test_non_2xx_raises_sdist_build_error(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + session = _make_session(b"", status=404) + candidate = _make_candidate("missing-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "download failed" in str(excinfo.value).lower() + assert "404" in str(excinfo.value) + + def test_no_response_raises_sdist_build_error(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + # Session.request raises → _http_request swallows and returns None. + session = MagicMock() + session.request.side_effect = ConnectionError("boom") + candidate = _make_candidate("offline-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "no response" in str(excinfo.value).lower() + + +# --------------------------------------------------------------------------- +# 4. Corrupt archive +# --------------------------------------------------------------------------- + + +class TestCorruptArchive: + def test_bogus_tarball_bytes_raise_corrupt(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + session = _make_session(b"this is not a tarball") + candidate = _make_candidate("garbage-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "corrupt" in str(excinfo.value).lower() + + def test_unknown_extension_raises_corrupt(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + session = _make_session(b"random bytes that aren't an archive") + candidate = _make_candidate("weird-1.0.weirdext") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "corrupt" in str(excinfo.value).lower() + + +# --------------------------------------------------------------------------- +# 5. Build backend failure (non-existent backend) +# --------------------------------------------------------------------------- + + +class TestBackendFailure: + def test_nonexistent_backend_raises(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + pyproject = ( + b"[build-system]\n" + b"requires = []\n" + b"build-backend = \"definitely_not_a_real_backend.module\"\n" + ) + sdist = _build_sdist_tarball( + pkg_name="badbackend", + version="0.0.1", + pyproject_toml=pyproject, + ) + session = _make_session(sdist) + candidate = _make_candidate("badbackend-0.0.1.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + msg = str(excinfo.value).lower() + assert "build backend failed" in msg + # The backend name should appear in the error so users can + # debug pyproject typos without re-extracting the sdist. + assert "definitely_not_a_real_backend.module" in str(excinfo.value) + + def test_malformed_pyproject_raises(self, tmp_path: Path): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + # Invalid TOML — unclosed string. + pyproject = b"[build-system\nrequires = [" + sdist = _build_sdist_tarball( + pkg_name="badtoml", + version="0.1", + pyproject_toml=pyproject, + ) + session = _make_session(sdist) + candidate = _make_candidate("badtoml-0.1.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "pyproject.toml" in str(excinfo.value) + + +# --------------------------------------------------------------------------- +# 6. No pyproject.toml → legacy fallback succeeds +# --------------------------------------------------------------------------- + + +class TestLegacyFallback: + def test_pyproject_with_no_build_backend_uses_legacy(self): + from pipenv.resolver.pure_python_sdist import ( + extract_metadata_from_sdist, + ) + + # pyproject.toml exists but declares no build-backend → PEP + # 517 §10 says use legacy setuptools. This is a separate + # branch in _resolve_build_backend from "no pyproject at all". + pyproject = b"[tool.somethingelse]\nkey = \"value\"\n" + sdist = _build_sdist_tarball( + pkg_name="legacypkg", + version="3.2.1", + pyproject_toml=pyproject, + ) + session = _make_session(sdist) + candidate = _make_candidate("legacypkg-3.2.1.tar.gz") + + result = extract_metadata_from_sdist(candidate, session) + assert result.name == "legacypkg" + assert result.version == "3.2.1" + + +# --------------------------------------------------------------------------- +# 7. Path traversal protection +# --------------------------------------------------------------------------- + + +class TestPathTraversalProtection: + def test_tar_member_with_dotdot_is_rejected(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + sdist = _build_traversal_tarball() + session = _make_session(sdist) + candidate = _make_candidate("malicious-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + msg = str(excinfo.value).lower() + # Either "traversal" wording from our validator, or "corrupt" + # if a later stage trips first — both surface a SdistBuildError + # which is the contract. + assert "corrupt" in msg or "traversal" in msg + + def test_absolute_path_member_is_rejected(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tf: + info = tarfile.TarInfo(name="/etc/passwd_steal") + payload = b"steal" + info.size = len(payload) + tf.addfile(info, io.BytesIO(payload)) + sdist = buf.getvalue() + session = _make_session(sdist) + candidate = _make_candidate("absroot-1.0.tar.gz") + + with pytest.raises(SdistBuildError): + extract_metadata_from_sdist(candidate, session) + + def test_zip_member_with_dotdot_is_rejected(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("../escape.txt", "pwned") + sdist = buf.getvalue() + session = _make_session(sdist) + candidate = _make_candidate("zipescape-1.0.zip") + + with pytest.raises(SdistBuildError): + extract_metadata_from_sdist(candidate, session) + + +# --------------------------------------------------------------------------- +# Extra coverage: extraction-shape edge cases +# --------------------------------------------------------------------------- + + +class TestExtractionShape: + def test_archive_with_no_top_level_dir_rejected(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + # A tarball that puts files at the root (no top-level dir) is + # not a well-formed sdist. + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tf: + info = tarfile.TarInfo(name="loose.txt") + payload = b"not in a directory" + info.size = len(payload) + tf.addfile(info, io.BytesIO(payload)) + sdist = buf.getvalue() + session = _make_session(sdist) + candidate = _make_candidate("flat-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "corrupt" in str(excinfo.value).lower() + + def test_archive_with_multiple_top_level_dirs_rejected(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tf: + for d in ("first-1.0", "second-1.0"): + info = tarfile.TarInfo(name=f"{d}/marker.txt") + payload = b"hi" + info.size = len(payload) + tf.addfile(info, io.BytesIO(payload)) + sdist = buf.getvalue() + session = _make_session(sdist) + candidate = _make_candidate("dual-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "exactly one" in str(excinfo.value).lower() + + +# --------------------------------------------------------------------------- +# Extra coverage: timeout via patched BuildBackendHookCaller +# --------------------------------------------------------------------------- + + +class TestTimeout: + def test_hook_caller_timeout_raises_sdist_build_error(self, monkeypatch): + """Patch the isolated-build helper to hang past the timeout.""" + from pipenv.resolver import pure_python_sdist + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + # Shrink the timeout dramatically so we don't pay 5 minutes. + monkeypatch.setattr(pure_python_sdist, "_BUILD_TIMEOUT_SECONDS", 0.2) + + def _hanging_build(source_root, metadata_dir): + import time + + time.sleep(5) + return Path(metadata_dir) + + monkeypatch.setattr( + pure_python_sdist, + "_build_metadata_in_isolated_env", + _hanging_build, + ) + + sdist = _build_sdist_tarball( + pkg_name="hangy", + version="0.1", + pyproject_toml=None, + ) + session = _make_session(sdist) + candidate = _make_candidate("hangy-0.1.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "timed out" in str(excinfo.value).lower() + + +# --------------------------------------------------------------------------- +# Extra coverage: filename fallback when candidate has no .filename +# --------------------------------------------------------------------------- + + +class TestFilenameFallback: + def test_candidate_without_filename_uses_url_tail(self): + from pipenv.resolver.pure_python_sdist import ( + extract_metadata_from_sdist, + ) + + sdist = _build_sdist_tarball( + pkg_name="urlpkg", + version="0.5", + pyproject_toml=None, + ) + session = _make_session(sdist) + candidate = SimpleNamespace( + url="https://example.test/sdists/urlpkg-0.5.tar.gz", + filename=None, + ) + + result = extract_metadata_from_sdist(candidate, session) + assert result.name == "urlpkg" + + def test_filename_from_url_strips_query_string(self): + from pipenv.resolver.pure_python_sdist import _filename_from_url + + assert ( + _filename_from_url("https://x.test/foo-1.0.tar.gz?token=abc") + == "foo-1.0.tar.gz" + ) + + def test_filename_from_url_handles_empty_tail(self): + from pipenv.resolver.pure_python_sdist import _filename_from_url + + # Trailing slash → empty tail → default fallback name. + assert _filename_from_url("https://x.test/path/") == "sdist.tar.gz" + + +# --------------------------------------------------------------------------- +# Extra coverage: defensive error branches +# --------------------------------------------------------------------------- + + +class TestDefensiveBranches: + def test_empty_response_body_raises(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + # Build a response whose .data attribute is None — _response_body + # returns None and the download step bails out cleanly. + response = MagicMock() + response.status = 200 + response.data = None + response.headers = {} + # Strip .content too so the body helper can't fall back to it. + del response.content + session = MagicMock() + session.request = MagicMock(return_value=response) + candidate = _make_candidate("empty-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "empty body" in str(excinfo.value) + + def test_pyproject_with_non_string_backend_falls_back_to_legacy( + self, tmp_path: Path + ): + from pipenv.resolver.pure_python_sdist import _resolve_build_backend + + # build-backend is an int — _resolve_build_backend should + # return the legacy fallback rather than raise. + (tmp_path / "pyproject.toml").write_bytes( + b"[build-system]\nrequires=[]\nbuild-backend = 0\n" + ) + backend, backend_path = _resolve_build_backend(tmp_path) + assert backend == "setuptools.build_meta:__legacy__" + assert backend_path is None + + def test_pyproject_with_backend_path_list_is_preserved( + self, tmp_path: Path + ): + from pipenv.resolver.pure_python_sdist import _resolve_build_backend + + (tmp_path / "pyproject.toml").write_bytes( + b'[build-system]\n' + b'requires=[]\n' + b'build-backend = "my.backend"\n' + b'backend-path = ["sub"]\n' + ) + (tmp_path / "sub").mkdir() + backend, backend_path = _resolve_build_backend(tmp_path) + assert backend == "my.backend" + assert backend_path == ["sub"] + + def test_pyproject_with_invalid_backend_path_type_is_ignored( + self, tmp_path: Path + ): + from pipenv.resolver.pure_python_sdist import _resolve_build_backend + + # backend-path is a string, not a list — we ignore it + # defensively rather than crashing. + (tmp_path / "pyproject.toml").write_bytes( + b'[build-system]\n' + b'requires=[]\n' + b'build-backend = "my.backend"\n' + b'backend-path = "not_a_list"\n' + ) + backend, backend_path = _resolve_build_backend(tmp_path) + assert backend == "my.backend" + assert backend_path is None + + def test_empty_member_name_in_zip_rejected(self): + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + # Hand-craft a zip with an empty filename entry. ZipFile lets + # us write that. + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("", b"empty name member") + session = _make_session(buf.getvalue()) + candidate = _make_candidate("emptyname-1.0.zip") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "corrupt" in str(excinfo.value).lower() + + def test_build_backend_returns_missing_metadata_file(self, monkeypatch): + """Patch the build helper to return a path with no METADATA.""" + from pipenv.resolver import pure_python_sdist + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + def _ghost_build(source_root, metadata_dir): + # Return a dist-info path that doesn't actually exist on + # disk. The post-call ``read_bytes`` will fail. + return Path(metadata_dir) / "ghost-1.0.dist-info" + + monkeypatch.setattr( + pure_python_sdist, + "_build_metadata_in_isolated_env", + _ghost_build, + ) + + sdist = _build_sdist_tarball( + pkg_name="ghost", + version="1.0", + pyproject_toml=None, + ) + session = _make_session(sdist) + candidate = _make_candidate("ghost-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "METADATA file not produced" in str(excinfo.value) + + def test_build_backend_emits_non_utf8_metadata(self, monkeypatch, tmp_path): + """Patch the build helper to produce a non-UTF-8 METADATA blob.""" + from pipenv.resolver import pure_python_sdist + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + def _bad_encoding_build(source_root, metadata_dir): + dist_info = Path(metadata_dir) / "bad-1.0.dist-info" + dist_info.mkdir(parents=True, exist_ok=True) + # 0xFF is not valid UTF-8 in a header position. + (dist_info / "METADATA").write_bytes(b"\xff\xfe not utf-8") + return dist_info + + monkeypatch.setattr( + pure_python_sdist, + "_build_metadata_in_isolated_env", + _bad_encoding_build, + ) + + sdist = _build_sdist_tarball( + pkg_name="bad", + version="1.0", + pyproject_toml=None, + ) + session = _make_session(sdist) + candidate = _make_candidate("bad-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "not" in str(excinfo.value).lower() and "utf-8" in str(excinfo.value).lower() + + def test_cache_write_oserror_is_non_fatal(self, monkeypatch, tmp_path): + """A failing cache write must not poison a successful build.""" + from pipenv.resolver.pure_python_metadata import MetadataCache + from pipenv.resolver.pure_python_sdist import ( + extract_metadata_from_sdist, + ) + + sdist = _build_sdist_tarball( + pkg_name="cachefail", + version="0.1", + pyproject_toml=None, + ) + session = _make_session(sdist) + candidate = _make_candidate("cachefail-0.1.tar.gz") + cache = MetadataCache(tmp_path / "metacache") + + def _explode(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr(cache, "put", _explode) + + # The result is still returned despite the cache.put OSError. + result = extract_metadata_from_sdist(candidate, session, cache=cache) + assert result.name == "cachefail" + + def test_archive_write_oserror_surfaces_as_sdist_build_error( + self, monkeypatch + ): + """Disk write failure during sdist download → SdistBuildError.""" + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + sdist = _build_sdist_tarball( + pkg_name="diskfull", + version="0.1", + pyproject_toml=None, + ) + session = _make_session(sdist) + candidate = _make_candidate("diskfull-0.1.tar.gz") + + real_write_bytes = Path.write_bytes + + def _explode(self: Path, data): + if self.name == "diskfull-0.1.tar.gz": + raise OSError("disk full") + return real_write_bytes(self, data) + + monkeypatch.setattr(Path, "write_bytes", _explode) + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + assert "could not write" in str(excinfo.value) + + def test_tar_member_with_device_type_is_rejected(self): + """Tar containing a block/char/fifo member → SdistBuildError.""" + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tf: + top = tarfile.TarInfo(name="evil-1.0/") + top.type = tarfile.DIRTYPE + tf.addfile(top) + dev = tarfile.TarInfo(name="evil-1.0/devnull") + dev.type = tarfile.CHRTYPE + dev.devmajor = 1 + dev.devminor = 3 + tf.addfile(dev) + + session = _make_session(buf.getvalue()) + candidate = _make_candidate("evil-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + msg = str(excinfo.value).lower() + assert "corrupt" in msg and "non-regular" in msg + + def test_tar_member_symlink_with_traversal_linkname_rejected(self): + """Tar symlink whose linkname escapes the root → rejected.""" + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + extract_metadata_from_sdist, + ) + + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tf: + top = tarfile.TarInfo(name="symlinky-1.0/") + top.type = tarfile.DIRTYPE + tf.addfile(top) + link = tarfile.TarInfo(name="symlinky-1.0/bad") + link.type = tarfile.SYMTYPE + link.linkname = "../../../etc/passwd" + tf.addfile(link) + + session = _make_session(buf.getvalue()) + candidate = _make_candidate("symlinky-1.0.tar.gz") + + with pytest.raises(SdistBuildError) as excinfo: + extract_metadata_from_sdist(candidate, session) + # Path-traversal validation catches the linkname. + assert "traversal" in str(excinfo.value).lower() or "absolute" in str(excinfo.value).lower() + + def test_archive_extracts_to_nothing_rejected(self, tmp_path): + """An archive that extracts to zero entries → SdistBuildError.""" + from pipenv.resolver.pure_python_sdist import ( + SdistBuildError, + _locate_source_root, + ) + + # _locate_source_root inspects the destination dir; we hand it + # an empty dir to exercise the "extracted to nothing" branch. + empty = tmp_path / "empty-extract" + empty.mkdir() + with pytest.raises(SdistBuildError) as excinfo: + _locate_source_root(empty, "empty-1.0.tar.gz") + assert "extracted to nothing" in str(excinfo.value) diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 6e5077a2d9..c7337c333f 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -166,6 +166,36 @@ def test_convert_to_pipenv_lockfile(pylock_file): assert lockfile["develop"]["pytest"]["markers"] == "'dev' in dependency_groups or 'test' in dependency_groups" +def test_convert_to_pipenv_lockfile_expands_env_vars_in_source_urls( + tmp_path, monkeypatch +): + """Regression test for gh-6670: pylock-converted sources must expand + ``${VAR}`` tokens in URLs just like ``Pipfile.lock`` reads do, otherwise + users with ``[pipenv] use_pylock = true`` see env-var auth silently + fall through to literal placeholders. + """ + pylock_path = tmp_path / "pylock.toml" + pylock_path.write_text( + 'lock-version = "1.0"\n' + 'created-by = "pipenv"\n' + "\n" + "[[sources]]\n" + 'name = "nexus"\n' + 'url = "https://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@nexus.example.com/repository/pypi/simple"\n' + "verify_ssl = true\n", + encoding="utf-8", + ) + monkeypatch.setenv("NEXUS_USERNAME", "alice") + monkeypatch.setenv("NEXUS_PASSWORD", "s3cret") + + pylock = PylockFile.from_path(pylock_path) + lockfile = pylock.convert_to_pipenv_lockfile() + sources = lockfile["_meta"]["sources"] + assert sources[0]["url"] == ( + "https://alice:s3cret@nexus.example.com/repository/pypi/simple" + ) + + def test_from_lockfile(tmp_path): """Test creating a PylockFile from a Pipfile.lock file.""" # Create a simple Pipfile.lock diff --git a/tests/unit/test_resolver_backends.py b/tests/unit/test_resolver_backends.py index 8c6d57762f..46a5d21244 100644 --- a/tests/unit/test_resolver_backends.py +++ b/tests/unit/test_resolver_backends.py @@ -225,3 +225,300 @@ def __init__(self, table): # And the absent case returns None (the default). settings_without = Settings(_FakeProject({})) assert settings_without.resolver is None + + +# --------------------------------------------------------------------------- +# T_PLUMBING (Initiative G phase 3): CLI flag + Pipfile-setting dispatcher +# --------------------------------------------------------------------------- +# +# T_F.5 wired the dispatcher inside :mod:`pipenv.resolver.core` and added +# the ``--resolver`` free-form CLI flag. T_PLUMBING adds the +# user-facing ``--backend`` flag (constrained ``choices=``), the +# ``[pipenv] resolver_backend`` Pipfile-setting accessor, and the +# parent-side precedence chain inside ``do_lock`` that stamps the +# resolved name onto the typed ``ResolverRequest.options.backend`` +# field via ``venv_resolve_deps(resolver_backend=...)``. +# +# These tests pin the four user-visible behaviours of T_PLUMBING: +# +# 1. ``--backend pure-python`` routes through ``state.resolver`` -> +# ``RoutineContext.execution_options.resolver``. +# 2. ``[pipenv] resolver_backend = "pure-python"`` is read by Settings +# and is preferred over the T_F.5 ``[pipenv] resolver`` back-compat +# alias. +# 3. CLI flag wins over Pipfile setting. +# 4. Default (neither set) yields ``None`` so the wire shape stays +# empty and the dispatcher's no-op default-path is byte-identical +# to today. + + +class TestPipfileResolverBackendSetting: + """``[pipenv] resolver_backend = "..."`` is the documented user-facing + Pipfile setting (T_PLUMBING). ``[pipenv] resolver`` is the T_F.5 + back-compat alias; both go through :class:`Settings` accessors. + """ + + def _fake_project(self, table): + class _FakePipfile: + def __init__(self, t): + self.parsed = {"pipenv": t} + + class _FakeProject: + def __init__(self, t): + self.pipfile = _FakePipfile(t) + + return _FakeProject(table) + + def test_resolver_backend_reads_pipenv_section(self): + from pipenv.utils.settings import Settings + + settings = Settings(self._fake_project({"resolver_backend": "pure-python"})) + assert settings.resolver_backend == "pure-python" + + def test_resolver_backend_absent_returns_none(self): + from pipenv.utils.settings import Settings + + settings = Settings(self._fake_project({})) + assert settings.resolver_backend is None + + def test_resolver_backend_whitespace_only_normalises_to_none(self): + from pipenv.utils.settings import Settings + + settings = Settings(self._fake_project({"resolver_backend": " "})) + assert settings.resolver_backend is None + + def test_resolver_backend_strips_surrounding_whitespace(self): + from pipenv.utils.settings import Settings + + settings = Settings(self._fake_project({"resolver_backend": " pip "})) + assert settings.resolver_backend == "pip" + + +class TestBackendCLIFlag: + """``--backend`` is a constrained alias the user-facing surface; it + flows through ``state.resolver`` for back-compat with the T_F.5 + ``--resolver`` free-form flag. The argparse ``choices=`` constraint + validates the value at parse time, so a typo fails with a clean + "invalid choice" error instead of falling through to the + dispatcher's ``InternalError`` translation. + """ + + def _parse_lock_args(self, argv): + from pipenv.cli.options import apply_env_vars, build_parser, build_state + + parser = build_parser() + ns = parser.parse_args(["lock", *argv]) + apply_env_vars(ns) + return ns + + def test_backend_flag_recognised_for_lock(self): + """``pipenv lock --backend pure-python`` parses cleanly.""" + ns = self._parse_lock_args(["--backend", "pure-python"]) + assert ns.backend == "pure-python" + + def test_backend_flag_rejects_unknown_choice(self, capsys): + """An unknown ``--backend`` value fails parsing with an + actionable "invalid choice" error. This is the T_PLUMBING + promise that typos don't silently fall through to the + dispatcher's KeyError path. + """ + import pytest + + with pytest.raises(SystemExit): + self._parse_lock_args(["--backend", "uv"]) # not in choices + err_text = capsys.readouterr().err + assert "invalid choice" in err_text or "argument" in err_text + + def test_backend_flag_default_is_none(self): + """No flag → ``ns.backend is None``. The downstream state / + routine chain then falls through to the Pipfile / env-var / + default chain — byte-identical to pre-T_PLUMBING behaviour. + """ + ns = self._parse_lock_args([]) + assert ns.backend is None + + def test_backend_flag_populates_state_resolver(self): + """``--backend NAME`` overrides ``--resolver NAME`` in + ``build_state`` (T_PLUMBING precedence: the constrained flag + wins over the free-form back-compat one). + """ + from pipenv.cli.options import apply_env_vars, build_parser, build_state + + parser = build_parser() + ns = parser.parse_args(["lock", "--backend", "pure-python"]) + apply_env_vars(ns) + state = build_state(ns) + assert state.resolver == "pure-python" + + def test_resolver_flag_alone_still_populates_state_resolver(self): + """Back-compat: ``--resolver NAME`` (T_F.5) still works. + ``--backend`` is the new spelling; ``--resolver`` is preserved. + """ + from pipenv.cli.options import apply_env_vars, build_parser, build_state + + parser = build_parser() + ns = parser.parse_args(["lock", "--resolver", "pip"]) + apply_env_vars(ns) + state = build_state(ns) + assert state.resolver == "pip" + + def test_backend_wins_over_resolver_when_both_set(self): + """When both flags are supplied, ``--backend`` (constrained) + wins over ``--resolver`` (free-form). Documented user surface + takes precedence over the back-compat alias. + """ + from pipenv.cli.options import apply_env_vars, build_parser, build_state + + parser = build_parser() + ns = parser.parse_args( + ["lock", "--resolver", "pip", "--backend", "pure-python"] + ) + apply_env_vars(ns) + state = build_state(ns) + assert state.resolver == "pure-python" + + def test_default_is_none(self): + """Neither flag → ``state.resolver is None`` → wire shape stays + empty → dispatcher falls through to default ``pip`` path. + """ + from pipenv.cli.options import apply_env_vars, build_parser, build_state + + parser = build_parser() + ns = parser.parse_args(["lock"]) + apply_env_vars(ns) + state = build_state(ns) + assert state.resolver is None + + +class TestVenvResolveDepsBackendPropagation: + """The ``resolver_backend`` kwarg on ``venv_resolve_deps`` reaches the + typed ``ResolverRequest.options.backend`` wire field via + ``_build_resolver_request``. This is the bridge from the routine + layer down to the subprocess argv / typed-request envelope. + """ + + def _fake_project(self): + """Build a stub ``project`` accepted by ``_build_resolver_request``. + + ``_build_resolver_request`` only touches ``project`` via the + deadline helper :func:`_resolve_deadline_seconds`, which reads + ``project.s.PIPENV_RESOLVER_TIMEOUT_S`` and + ``project.settings.get("resolver_timeout")``. A bare object + with those attributes is enough for our use. + """ + class _FakeSettings: + def get(self, _key, default=None): + return default + + class _S: + PIPENV_RESOLVER_TIMEOUT_S = 0 + + class _FakeProject: + s = _S() + settings = _FakeSettings() + + return _FakeProject() + + def test_build_resolver_request_stamps_backend(self): + from pipenv.utils.resolver import _build_resolver_request + + request = _build_resolver_request( + deps={"requests": "requests==2.31.0"}, + sources=[{"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True}], + category="default", + pre=False, + clear=False, + allow_global=False, + verbose=False, + python_marker_override=None, + extra_pip_args=[], + resolved_default_deps=None, + project=self._fake_project(), + resolver_backend="pure-python", + ) + assert request.options.backend == "pure-python" + + def test_build_resolver_request_defaults_backend_to_pip(self): + """No ``resolver_backend`` kwarg + no env / Pipfile override → + ``"pip"``. Per commit ``0bf0c192`` + (``fix(resolver): stamp selected backend onto resolver requests``), + the parent now resolves the full + precedence chain (CLI/caller > env > Pipfile > default) and + stamps the result onto the request before sending — so the + subprocess no longer has to rediscover the project's Pipfile to + make the same decision. The default-fallback value is ``"pip"``. + """ + from pipenv.utils.resolver import _build_resolver_request + + request = _build_resolver_request( + deps={"requests": "requests==2.31.0"}, + sources=[{"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True}], + category="default", + pre=False, + clear=False, + allow_global=False, + verbose=False, + python_marker_override=None, + extra_pip_args=[], + resolved_default_deps=None, + project=self._fake_project(), + ) + assert request.options.backend == "pip" + # The wire dict carries the resolved backend. + wire = request.to_json_dict() + assert wire["options"].get("backend") == "pip" + + +class TestResolverSubprocessDispatch: + """End-to-end: ``resolve_for_pipenv`` reads ``request.options.backend`` + and dispatches through ``get_backend(name).resolve(request)``. This + test confirms the dispatcher honours the parent-stamped selection + without consulting env / Pipfile fallbacks. + """ + + def test_subprocess_dispatches_to_named_backend(self): + from pipenv.resolver import core + from pipenv.resolver.backends import base as backends_base + + captured = {"called": False, "request": None} + + class _SentinelBackend: + name = "sentinel" + + def is_available(self) -> bool: + return True + + def resolve(self, request): + captured["called"] = True + captured["request"] = request + return ResolverResponse( + schema_version=SCHEMA_VERSION, + result=ResolverSuccess(kind="success", locked=()), + ) + + request = _build_request(options=ResolverOptions(backend="sentinel")) + + with mock.patch.dict( + backends_base.REGISTRY, {"sentinel": _SentinelBackend()} + ): + response = core.resolve_for_pipenv(request) + + assert captured["called"] is True + assert captured["request"] is request + assert isinstance(response, ResolverResponse) + assert isinstance(response.result, ResolverSuccess) + + def test_pure_python_backend_is_registered_and_dispatchable(self): + """The actual ``pure-python`` registry entry is reachable via + the dispatcher. We don't run a real resolve (that's T15's + integration job); we just confirm ``get_backend("pure-python")`` + returns an instance that ``resolve_for_pipenv`` would invoke. + """ + from pipenv.resolver.backends import REGISTRY, get_backend + from pipenv.resolver.backends.pure_python import PurePythonBackend + + assert "pure-python" in REGISTRY + backend = get_backend("pure-python") + assert isinstance(backend, PurePythonBackend) + # And it self-reports its name as the dispatcher would expect. + assert backend.name == "pure-python" diff --git a/tests/unit/test_resolver_parent_dispatch.py b/tests/unit/test_resolver_parent_dispatch.py index 54f589f46d..2e87f5f269 100644 --- a/tests/unit/test_resolver_parent_dispatch.py +++ b/tests/unit/test_resolver_parent_dispatch.py @@ -247,6 +247,42 @@ def test_request_stamps_pipfile_selected_backend(self, tmp_path, monkeypatch): ) assert request.options.backend == "uv" + def test_cache_key_uses_effective_backend_from_env(self, tmp_path, monkeypatch): + """When ``resolver_backend`` is omitted, the cache key still uses + the effective backend selected from ``PIPENV_RESOLVER``. + """ + project = _stub_project(tmp_path) + monkeypatch.setenv("PIPENV_RESOLVER", "pure-python") + + observed = {} + + def _fake_generate_cache_key(*_args, **kwargs): + observed["resolver_backend"] = kwargs.get("resolver_backend") + return "cache-key" + + monkeypatch.setattr( + resolver_mod, "_generate_resolution_cache_key", _fake_generate_cache_key + ) + monkeypatch.setattr(resolver_mod, "_should_use_resolution_cache", lambda *_a, **_k: True) + monkeypatch.setattr(resolver_mod, "prepare_lockfile", lambda *_a, **_k: {}) + + resolver_mod._resolution_cache["cache-key"] = [] + try: + resolver_mod.venv_resolve_deps( + deps={"requests": "*"}, + which=lambda *_a, **_k: "python", + project=project, + pipfile_category="packages", + pipfile={}, + lockfile={"default": {}}, + old_lock_data={}, + resolver_backend=None, + ) + finally: + resolver_mod._resolution_cache.pop("cache-key", None) + + assert observed["resolver_backend"] == "pure-python" + # --------------------------------------------------------------------------- # Subprocess invocation argv shape