Fix memory leaks on error in ComputeGainMap() (#3336)

The "Unsupported RGB color space" path and the avifImageScale() failure
path returned directly instead of going through the cleanup label,
leaking the gainMapF[] channel buffers and the gainMapRGB pixels. Both
now set res and jump to cleanup, as documented by the comment above the
allocations.

The regression test computes a gain map with an unsupported depth (only
rejected after the allocations) and checks that the gain map image
planes are freed on error. Verified with ASan/LSan.

Co-authored-by: krishna28238-arch <319297638+krishna28238-arch@users.noreply.github.com>

Author: krishna28238-arch

src/gainmap.c

@@ -804,7 +804,8 @@ avifResult avifRGBImageComputeGainMap(const avifRGBImage * baseRgbImage,
     avifRGBColorSpaceInfo gainMapRGBInfo;
     if (!avifGetRGBColorSpaceInfo(&gainMapRGB, &gainMapRGBInfo)) {
         avifDiagnosticsPrintf(diag, "Unsupported RGB color space");
-        return AVIF_RESULT_NOT_IMPLEMENTED;
+        res = AVIF_RESULT_NOT_IMPLEMENTED;
+        goto cleanup;
     }
     for (uint32_t j = 0; j < height; ++j) {
         for (uint32_t i = 0; i < width; ++i) {
@@ -825,7 +826,10 @@ avifResult avifRGBImageComputeGainMap(const avifRGBImage * baseRgbImage,
     // Scale down the gain map if requested.
     // Another way would be to scale the source images, but it seems to perform worse.
     if (requestedWidth != gainMapImage->width || requestedHeight != gainMapImage->height) {
-        AVIF_CHECKRES(avifImageScale(gainMap->image, requestedWidth, requestedHeight, diag));
+        res = avifImageScale(gainMap->image, requestedWidth, requestedHeight, diag);
+        if (res != AVIF_RESULT_OK) {
+            goto cleanup;
+        }
     }
 
 cleanup:

tests/gtest/avifgainmaptest.cc

@@ -1593,6 +1593,32 @@ TEST(GainMapTest, CreateGainMapConstantFactor) {
       << avifResultToString(result) << " " << diag.error;
 }
 
+// avifRGBImageComputeGainMap() should free all its intermediate buffers when
+// it fails, in particular when the gain map depth is only rejected after the
+// allocations happened.
+TEST(GainMapTest, ComputeGainMapUnsupportedDepth) {
+  // Used only to initialize rgb images.
+  ImagePtr yuv(avifImageCreate(10, 10, 8, AVIF_PIXEL_FORMAT_YUV444));
+  testutil::AvifRgbImage base_image(yuv.get(), 8, AVIF_RGB_FORMAT_RGB);
+  testutil::AvifRgbImage alt_image(yuv.get(), 8, AVIF_RGB_FORMAT_RGB);
+  for (uint32_t i = 0; i < base_image.width * base_image.height * 3; ++i) {
+    base_image.pixels[i] = 10;
+    alt_image.pixels[i] = 200;
+  }
+  GainMapPtr gain_map(avifGainMapCreate());
+  gain_map->image = avifImageCreate(5, 5, 9, AVIF_PIXEL_FORMAT_YUV444);
+  avifDiagnostics diag;
+  avifResult result = avifRGBImageComputeGainMap(
+      &base_image, AVIF_COLOR_PRIMARIES_SRGB,
+      AVIF_TRANSFER_CHARACTERISTICS_SRGB, &alt_image, AVIF_COLOR_PRIMARIES_SRGB,
+      AVIF_TRANSFER_CHARACTERISTICS_SRGB, gain_map.get(), &diag);
+
+  EXPECT_EQ(result, AVIF_RESULT_NOT_IMPLEMENTED)
+      << avifResultToString(result) << " " << diag.error;
+  // The planes are freed on error, per the cleanup contract.
+  EXPECT_EQ(gain_map->image->yuvPlanes[0], nullptr);
+}
+
 TEST(FindMinMaxWithoutOutliers, AllSame) {
   constexpr int kNumValues = 10000;