fix(ci): benchmarks UTF-8 and pre-e2e test scope

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: davidberenstein1957

src/pruna/evaluation/benchmarks.py

@@ -285,13 +285,6 @@ def list(cls, task_type: str | None = None) -> list[str]:
         task_type="text_to_image",
         reference="https://arxiv.org/abs/2506.07977",
     ),
-    Benchmark(
-        name="OneIG Knowledge Reasoning",
-        description="OneIG subset: knowledge- and reasoning-heavy prompts.",
-        metrics=["oneig_reasoning"],
-        task_type="text_to_image",
-        reference="https://arxiv.org/abs/2506.07977",
-    ),
     Benchmark(
         name="OneIG Multilingualism",
         description="OneIG subset: multilingual prompts (incl. Chinese splits).",

tests/evaluation/test_text_metrics.py

@@ -1,4 +1,4 @@
-"""Tests for OneIG alignment (masking, wiring) and OneIG reasoning (LLM2CLIP)."""
+"""Tests for OneIG alignment masking and wiring."""
 
 from unittest.mock import MagicMock
 
@@ -12,11 +12,6 @@
     aggregate_oneig_alignment_per_cell,
     apply_oneig_dependency_mask,
 )
-from pruna.evaluation.metrics.metric_oneig_reasoning import (
-    OneIGReasoningMetric,
-    _LLM2CLIPScorer,
-)
-from pruna.evaluation.metrics.registry import MetricRegistry
 from pruna.evaluation.metrics.vlm_base import BaseVLM
 
 
@@ -125,107 +120,3 @@ def test_oneig_alignment_all_padding_questions_yields_zero_without_vlm() -> None
     assert metric.compute().result == 0.0
     mock_vlm.score.assert_not_called()
 
-
-def test_to_oneig_record_strips_null_questions_and_dependencies() -> None:
-    """Null-valued Q_D entries are filtered out at record construction time."""
-    row = {"category": "Anime_Stylization", "id": "001", "class": "None", "prompt_en": "a cat"}
-    questions_by_key = {
-        "anime_001": {
-            "questions": {"1": "Is there a cat?", "21": None},
-            "dependencies": {"1": [0], "21": None},
-        }
-    }
-    record = _to_oneig_record(row, questions_by_key, {}, {})
-    assert "21" not in record["questions"]
-    assert "21" not in record["dependencies"]
-    assert record["questions"] == {"1": "Is there a cat?"}
-    assert record["dependencies"] == {"1": [0]}
-
-
-def _make_mock_scorer(return_value: float = 0.5) -> MagicMock:
-    mock = MagicMock(spec=_LLM2CLIPScorer)
-    mock.score.return_value = [return_value]
-    return mock
-
-
-@pytest.mark.cpu
-def test_oneig_reasoning_uses_gt_answer_from_aux() -> None:
-    """Metric reads reasoning_gt_answer from aux."""
-    mock_scorer = _make_mock_scorer(0.7)
-    metric = OneIGReasoningMetric(scorer=mock_scorer, device="cpu")
-    images = torch.rand(1, 3, 64, 64)
-    aux = {"reasoning_gt_answer": "A blue circle"}
-    metric.update(["p"], [aux], images)
-    result = metric.compute()
-    assert result.name == "oneig_reasoning"
-    assert result.result == 0.7
-    mock_scorer.score.assert_called_once()
-    call_args = mock_scorer.score.call_args
-    assert call_args[0][1] == "A blue circle"
-
-
-@pytest.mark.cpu
-def test_oneig_reasoning_averages_per_sample_scores() -> None:
-    """Compute returns mean of per-sample scores."""
-    mock_scorer = _make_mock_scorer(0.5)
-    metric = OneIGReasoningMetric(scorer=mock_scorer, device="cpu")
-    images = torch.rand(2, 3, 64, 64)
-    aux_list = [
-        {"reasoning_gt_answer": "First answer"},
-        {"reasoning_gt_answer": "Second answer"},
-    ]
-    metric.update(["p1", "p2"], aux_list, images)
-    result = metric.compute()
-    assert result.result == 0.5
-    assert mock_scorer.score.call_count == 2
-
-
-@pytest.mark.cpu
-def test_oneig_reasoning_missing_gt_raises() -> None:
-    """Missing GT answer raises ValueError."""
-    mock_scorer = _make_mock_scorer(0.8)
-    metric = OneIGReasoningMetric(scorer=mock_scorer, device="cpu")
-    images = torch.rand(1, 3, 64, 64)
-    aux = {}
-    with pytest.raises(ValueError, match="reasoning_gt_answer"):
-        metric.update(["p"], [aux], images)
-    mock_scorer.score.assert_not_called()
-
-
-@pytest.mark.cpu
-def test_oneig_reasoning_scorer_none_raises() -> None:
-    """When scorer returns None, metric raises RuntimeError."""
-    mock_scorer = _make_mock_scorer()
-    mock_scorer.score.return_value = None
-    metric = OneIGReasoningMetric(scorer=mock_scorer, device="cpu")
-    images = torch.rand(1, 3, 64, 64)
-    aux = {"reasoning_gt_answer": "Some answer"}
-    with pytest.raises(RuntimeError, match="no scores"):
-        metric.update(["p"], [aux], images)
-
-
-@pytest.mark.cpu
-def test_oneig_reasoning_compute_without_update_raises() -> None:
-    """Compute with no updates raises RuntimeError."""
-    mock_scorer = _make_mock_scorer()
-    metric = OneIGReasoningMetric(scorer=mock_scorer, device="cpu")
-    with pytest.raises(RuntimeError, match="no samples were scored"):
-        metric.compute()
-
-
-@pytest.mark.cpu
-def test_oneig_reasoning_has_metric_registered() -> None:
-    """oneig_reasoning is available via MetricRegistry (lazy)."""
-    assert MetricRegistry.has_metric("oneig_reasoning")
-
-
-@pytest.mark.cpu
-def test_transformers_major_version_supported_for_oneig_reasoning() -> None:
-    """Enforce pyproject ``transformers<5`` expectation for LLM2CLIP loading."""
-    import transformers
-
-    major = int(transformers.__version__.split(".", 1)[0])
-    assert major < 5, (
-        "oneig_reasoning expects transformers 4.x (see pyproject.toml); "
-        "5.x from_pretrained buffer initialization can break CLIP/Llama stacks."
-    )