feat: clipiqa metric (#259)

* feat: add CLIP Image Quality Assessment metric and update dependencies

* Introduced `CLIPImageQualityAssessment` to the TorchMetrics enumeration.
* Updated `MetricResult` to support results as a list for IQA metrics.
* Added a new test for the `clipiqa` metric to validate its functionality.
* Included `piq` as a dependency in `pyproject.toml`.

* fix: update pairwise metric handling in tests

* Modified the test for `TorchMetricWrapper` to include `clipiqa` alongside `arniqa` for pairwise call types.
* Adjusted assertions to ensure correct call type behavior for both metrics.

---------

Co-authored-by: davidberenstein1957 <david.m.berenstein@gmail.com>

Author: begumcig

pyproject.toml

@@ -115,7 +115,8 @@ dependencies = [
     "hqq==0.2.6",
     "torchao",
     "llmcompressor",
-    "gliner; python_version >= '3.10'"
+    "gliner; python_version >= '3.10'",
+    "piq",
 
 ]
 

src/pruna/evaluation/metrics/metric_torch.py

@@ -31,6 +31,7 @@
     StructuralSimilarityIndexMeasure,
 )
 from torchmetrics.image.arniqa import ARNIQA
+from torchmetrics.multimodal.clip_iqa import CLIPImageQualityAssessment
 from torchmetrics.multimodal.clip_score import CLIPScore
 from torchmetrics.text import Perplexity
 from torchvision import transforms
@@ -167,11 +168,7 @@ class TorchMetrics(Enum):
     """
 
     fid = (partial(FrechetInceptionDistance), fid_update, "gt_y")
-    accuracy = (
-        partial(Accuracy),
-        None,
-        "y_gt",
-    )
+    accuracy = (partial(Accuracy), None, "y_gt")
     perplexity = (partial(Perplexity), None, "y_gt")
     clip_score = (partial(CLIPScore), None, "y_x")
     precision = (partial(Precision), None, "y_gt")
@@ -180,6 +177,7 @@ class TorchMetrics(Enum):
     ssim = (partial(StructuralSimilarityIndexMeasure), ssim_update, "pairwise_y_gt")
     lpips = (partial(LearnedPerceptualImagePatchSimilarity), lpips_update, "pairwise_y_gt")
     arniqa = (partial(ARNIQA), arniqa_update, "y")
+    clipiqa = (partial(CLIPImageQualityAssessment), None, "y")
 
     def __init__(self, *args, **kwargs) -> None:
         self.tm = self.value[0]
@@ -341,10 +339,17 @@ def compute(self) -> Any:
             The computed metric value.
         """
         result = self.metric.compute()
+
+        # Normally we have a single score for each metric for the entire dataset.
+        # For IQA metrics we have a single score per image, so we need to convert the tensor to a list.
+        if isinstance(result, Tensor):
+            result_value = result.item() if result.numel() == 1 else result.tolist()
+        else:
+            result_value = result
         return MetricResult(
             self.metric_name,
             self.__dict__.copy(),
-            result.item() if isinstance(result, Tensor) else result,
+            result_value,
         )
 
 

src/pruna/evaluation/metrics/result.py

@@ -14,7 +14,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 
 @dataclass
@@ -34,7 +34,7 @@ class MetricResult:
 
     name: str
     params: Dict[str, Any]
-    result: float | int
+    result: float | int | List[float | int]
 
     def __str__(self) -> str:
         """

tests/evaluation/test_torch_metrics.py

@@ -52,6 +52,16 @@ def test_clip_score(dataloader_fixture: Any) -> None:
     score = metric.compute()
     assert score.result > 0.0 and score.result < 100.0
 
+@pytest.mark.cpu
+@pytest.mark.parametrize("dataloader_fixture", ["LAION256"], indirect=True)
+def test_clipiqa(dataloader_fixture: Any) -> None:
+    """Test the clipiqa."""
+    metric = TorchMetricWrapper("clipiqa")
+    x, gt = next(iter(dataloader_fixture))
+    metric.update(x, gt, gt)
+    score = metric.compute()
+    assert score.result > 0.0 and score.result < 1.0
+
 
 @pytest.mark.cpu
 @pytest.mark.parametrize("dataloader_fixture", ["ImageNet"], indirect=True)
@@ -79,14 +89,14 @@ def test_check_call_type(metric: str, call_type: str):
     kwargs = {}
     if metric in ['accuracy', 'recall', 'precision']:
         kwargs = {"task": "multiclass", "num_classes": 1000}
-    if metric == "arniqa" and call_type == "pairwise":
+    if metric in ["arniqa", "clipiqa"] and call_type == "pairwise":
         with pytest.raises(Exception):
             TorchMetricWrapper(metric, call_type=call_type, **kwargs)
         return
     metric = TorchMetricWrapper(metric, call_type=call_type, **kwargs)
-    if call_type == "pairwise" and metric.metric_name != "arniqa":
+    if call_type == "pairwise" and metric.metric_name not in ["arniqa", "clipiqa"]:
         assert metric.call_type.startswith("pairwise")
-    elif metric.metric_name == "arniqa":
+    elif metric.metric_name in ["arniqa", "clipiqa"]:
         assert metric.call_type == "y"
     else:
         assert not metric.call_type.startswith("pairwise")