Address review comments round 2

Author: maryla-uc

apps/avifgainmaputil/imageio.cc

@@ -40,25 +40,24 @@ static avifResult UpsampleYUV420To444(avifImage* image) {
   const uint32_t old_uv_height = (uint32_t)(((uint64_t)image->height + 1) / 2);
   const uint32_t bytes_per_pixel = (image->depth > 8) ? 2 : 1;
 
-  const uint64_t uv_byte_size =
-      (uint64_t)new_uv_width * new_uv_height * bytes_per_pixel;
-  if (uv_byte_size > SIZE_MAX) {
+  if (new_uv_width > UINT32_MAX / bytes_per_pixel) {
     return AVIF_RESULT_INVALID_ARGUMENT;
   }
-  uint8_t* new_u = (uint8_t*)avifAlloc((size_t)uv_byte_size);
-  uint8_t* new_v = (uint8_t*)avifAlloc((size_t)uv_byte_size);
+  const uint32_t new_stride_u = new_uv_width * bytes_per_pixel;
+  const uint32_t new_stride_v = new_stride_u;
+
+  if (new_stride_u != 0 && new_uv_height > SIZE_MAX / new_stride_u) {
+    return AVIF_RESULT_INVALID_ARGUMENT;
+  }
+  const size_t uv_byte_size = (size_t)new_stride_u * new_uv_height;
+  uint8_t* new_u = (uint8_t*)avifAlloc(uv_byte_size);
+  uint8_t* new_v = (uint8_t*)avifAlloc(uv_byte_size);
   if (!new_u || !new_v) {
     avifFree(new_u);
     avifFree(new_v);
     return AVIF_RESULT_OUT_OF_MEMORY;
   }
 
-  if (UINT32_MAX / bytes_per_pixel < new_uv_width) {
-    return AVIF_RESULT_INVALID_ARGUMENT;
-  }
-  const uint32_t new_stride_u = new_uv_width * bytes_per_pixel;
-  const uint32_t new_stride_v = new_uv_width * bytes_per_pixel;
-
 #if defined(AVIF_LIBYUV_ENABLED)
   if (image->depth == 8) {
     libyuv::ScalePlane(image->yuvPlanes[AVIF_CHAN_U],
@@ -149,22 +148,18 @@ static avifResult DownsampleYUV444To420(avifImage* image) {
   const uint32_t new_uv_height = (uint32_t)(((uint64_t)image->height + 1) / 2);
   const uint32_t bytes_per_pixel = (image->depth > 8) ? 2 : 1;
 
-  const uint64_t uv_byte_size =
-      (uint64_t)new_uv_width * new_uv_height * bytes_per_pixel;
-  if (uv_byte_size > SIZE_MAX) {
-    return AVIF_RESULT_INVALID_ARGUMENT;
-  }
-  uint8_t* new_u = (uint8_t*)avifAlloc((size_t)uv_byte_size);
-  uint8_t* new_v = (uint8_t*)avifAlloc((size_t)uv_byte_size);
+  const uint32_t new_stride_u = new_uv_width * bytes_per_pixel;
+  const uint32_t new_stride_v = new_stride_u;
+
+  const size_t uv_byte_size = (size_t)new_stride_u * new_uv_height;
+  uint8_t* new_u = (uint8_t*)avifAlloc(uv_byte_size);
+  uint8_t* new_v = (uint8_t*)avifAlloc(uv_byte_size);
   if (!new_u || !new_v) {
     avifFree(new_u);
     avifFree(new_v);
     return AVIF_RESULT_OUT_OF_MEMORY;
   }
 
-  const uint32_t new_stride_u = new_uv_width * bytes_per_pixel;
-  const uint32_t new_stride_v = new_uv_width * bytes_per_pixel;
-
 #if defined(AVIF_LIBYUV_ENABLED)
   if (image->depth == 8) {
     libyuv::ScalePlane(image->yuvPlanes[AVIF_CHAN_U],