Replace AVIF_ASSERT by avm image->depth condition (#3046)

Disable 12-bit AV2 tests.

Author: y-guyon

src/codec_avm.c

@@ -510,9 +510,10 @@ static avifResult avmCodecEncodeImage(avifCodec * codec,
         //            12-bit 4:0:0, 4:2:0, 4:2:2 and 4:4:4
         uint8_t seqProfile = 0;
 #if defined(CONFIG_AV2_PROFILES) && CONFIG_AV2_PROFILES
-        // Only 8-bit and 10-bit are supported.
-        AVIF_ASSERT_OR_RETURN(image->depth == 8 || image->depth == 10);
-
+        if (image->depth != 8 && image->depth != 10) {
+            avifDiagnosticsPrintf(codec->diag, "%d-bit is not supported in AV2 encoder.", image->depth);
+            return AVIF_RESULT_INVALID_ARGUMENT;
+        }
         // Based on https://gitlab.com/AOMediaCodec/avm/-/blob/main/av2/common/enums.h?ref_type=fcab0163f471b38fe593672fcbd24a6beb0be82e#L272
         if (alpha) {
             seqProfile = 3; // Main_420_10

tests/gtest/avifavmtest.cc

@@ -71,13 +71,29 @@ INSTANTIATE_TEST_SUITE_P(Basic, AvmTest,
 
 INSTANTIATE_TEST_SUITE_P(Tiny, AvmTest,
                          Combine(/*width=*/Values(1), /*height=*/Values(1),
-                                 /*depth=*/Values(8, 10, 12),
+                                 /*depth=*/Values(8, 10),
+                                 Values(AVIF_PIXEL_FORMAT_YUV444),
+                                 /*alpha=*/Values(false)));
+
+// avm does not support 12-bit as of today.
+INSTANTIATE_TEST_SUITE_P(DISABLED_Tiny12bit, AvmTest,
+                         Combine(/*width=*/Values(1), /*height=*/Values(1),
+                                 /*depth=*/Values(12),
                                  Values(AVIF_PIXEL_FORMAT_YUV444),
                                  /*alpha=*/Values(false)));
 
 INSTANTIATE_TEST_SUITE_P(HighBitDepthAndEvenDimensions, AvmTest,
                          Combine(/*width=*/Values(5), /*height=*/Values(34),
-                                 /*depth=*/Values(10, 12),
+                                 /*depth=*/Values(10),
+                                 Values(AVIF_PIXEL_FORMAT_YUV400,
+                                        AVIF_PIXEL_FORMAT_YUV420,
+                                        AVIF_PIXEL_FORMAT_YUV444),
+                                 /*alpha=*/Values(true)));
+
+// avm does not support 12-bit as of today.
+INSTANTIATE_TEST_SUITE_P(DISABLED_HighBitDepthAndEvenDimensions12bit, AvmTest,
+                         Combine(/*width=*/Values(5), /*height=*/Values(34),
+                                 /*depth=*/Values(12),
                                  Values(AVIF_PIXEL_FORMAT_YUV400,
                                         AVIF_PIXEL_FORMAT_YUV420,
                                         AVIF_PIXEL_FORMAT_YUV444),