Add support for handling 'tmap' items

Gain map technology offers a way to create an image (henceforth referred
as derived image) from one base input image and a secondary input image
called as a gainmap input image. Reconstruction is done by applying the
gain map to the base image according to iso 21496-1 section 6.

This change provides API to read, write base input image, gain map input
image, gain map metadata. Third party applications can make use of this
API to render only base image or reconstruct the derived image and
render the same.

By default these API is disabled and can be enabled at configure time
using option -DWITH_EXPERIMENTAL_GAIN_MAP=1

TODO: add plugins for encoding, decoding uhdr images

Author: ram-mohan

CMakePresets.json

@@ -54,6 +54,7 @@
         "WITH_VVDEC_PLUGIN" : "OFF",
         "WITH_VVENC" : "ON",
         "WITH_VVENC_PLUGIN" : "OFF",
+        "WITH_EXPERIMENTAL_GAIN_MAP" : "OFF",
 
         "WITH_REDUCED_VISIBILITY" : "OFF",
         "WITH_HEADER_COMPRESSION" : "ON",
@@ -110,6 +111,7 @@
         "WITH_VVDEC_PLUGIN" : "ON",
         "WITH_VVENC" : "ON",
         "WITH_VVENC_PLUGIN" : "ON",
+        "WITH_EXPERIMENTAL_GAIN_MAP" : "OFF",
 
         "WITH_REDUCED_VISIBILITY" : "ON",
         "WITH_HEADER_COMPRESSION" : "ON",
@@ -148,6 +150,7 @@
         "WITH_UVG266" : "OFF",
         "WITH_VVDEC" : "OFF",
         "WITH_VVENC" : "OFF",
+        "WITH_EXPERIMENTAL_GAIN_MAP" : "OFF",
 
         "WITH_REDUCED_VISIBILITY" : "ON",
         "WITH_HEADER_COMPRESSION" : "OFF",

libheif/CMakeLists.txt

@@ -249,6 +249,10 @@ if (ENABLE_EXPERIMENTAL_MINI_FORMAT)
             mini.cc)
 endif ()
 
+if (WITH_EXPERIMENTAL_GAIN_MAP)
+    target_compile_definitions(heif PUBLIC WITH_EXPERIMENTAL_GAIN_MAP=1)
+endif ()
+
 write_basic_package_version_file(${PROJECT_NAME}-config-version.cmake COMPATIBILITY ExactVersion)
 
 install(TARGETS heif EXPORT ${PROJECT_NAME}-config

libheif/api/libheif/heif.cc

@@ -657,6 +657,86 @@ struct heif_error heif_context_get_primary_image_ID(struct heif_context* ctx, he
 }
 
 
+#if WITH_EXPERIMENTAL_GAIN_MAP
+heif_error heif_context_get_gain_map_image_handle(heif_context* ctx, heif_image_handle** img) {
+  if (!img) {
+    Error err(heif_error_Usage_error, heif_suberror_Null_pointer_argument);
+    return err.error_struct(ctx->context.get());
+  }
+
+  std::shared_ptr<ImageItem> gain_map_image = ctx->context->get_gain_map_image();
+  if (!gain_map_image) {
+    Error err(heif_error_Invalid_input, heif_suberror_No_item_data);
+    return err.error_struct(ctx->context.get());
+  }
+
+  *img = new heif_image_handle();
+  (*img)->image = std::move(gain_map_image);
+  (*img)->context = ctx->context;
+
+  return Error::Ok.error_struct(ctx->context.get());
+}
+
+// As per ISO/IEC 23008-12, Support for tone map derived items and other improvements, Section:
+// Tone-map derivation, A 'tmap' derived item shall be associated with a 'colr' item property. This
+// need not be cicp but also icc. TODO: Add support for handling raw (icc) color profile
+struct heif_error heif_context_get_tmap_nclx_color_profile(
+    struct heif_context* ctx, struct heif_color_profile_nclx** out_data) {
+  if (!out_data) {
+    Error err(heif_error_Usage_error, heif_suberror_Null_pointer_argument);
+    return err.error_struct(ctx->context.get());
+  }
+
+  Error err = ctx->context->get_tmap_color_profile(out_data);
+  return err.error_struct(ctx->context.get());
+}
+
+size_t heif_context_get_gain_map_metadata_size(struct heif_context* ctx) {
+  std::shared_ptr<ImageMetadata> metadata = ctx->context->get_gain_map_metadata();
+
+  if (metadata) {
+    // Ignore unsigned int(8) version = 0 field of ToneMapImage syntax
+    size_t sz = metadata->m_data.size();
+    return sz > 0 ? (sz - 1) : 0;
+  }
+
+  return 0;
+}
+
+struct heif_error heif_context_get_gain_map_metadata(struct heif_context* ctx, void* out_data) {
+  if (!out_data) {
+    Error err(heif_error_Usage_error, heif_suberror_Null_pointer_argument);
+    return err.error_struct(ctx->context.get());
+  }
+
+  std::shared_ptr<ImageMetadata> metadata = ctx->context->get_gain_map_metadata();
+  if (!metadata) {
+    Error err(heif_error_Invalid_input, heif_suberror_No_item_data);
+    return err.error_struct(ctx->context.get());
+  }
+
+  uint8_t version = 0xff;
+  size_t pos = 0;
+  std::vector<uint8_t>& buffer = metadata->m_data;
+
+  if (pos >= buffer.size()) {
+    Error err(heif_error_Invalid_input, heif_suberror_End_of_data);
+    return err.error_struct(ctx->context.get());
+  }
+  version = buffer[pos++];
+  if (version != 0) {
+    Error err(heif_error_Invalid_input, heif_suberror_Unsupported_data_version,
+              "Box[tmap] has unsupported version");
+    return err.error_struct(ctx->context.get());
+  }
+
+  memcpy(out_data, buffer.data() + pos, buffer.size() - pos);
+
+  return heif_error_success;
+}
+#endif
+
+
 int heif_context_is_top_level_image_ID(struct heif_context* ctx, heif_item_id id)
 {
   const std::vector<std::shared_ptr<ImageItem>> images = ctx->context->get_top_level_images(true);
@@ -3442,6 +3522,117 @@ struct heif_error heif_context_encode_image(struct heif_context* ctx,
 }
 
 
+#if WITH_EXPERIMENTAL_GAIN_MAP
+struct heif_error heif_context_encode_gain_map_image(
+    struct heif_context* ctx, const struct heif_image* alternate_image,
+    const struct heif_image_handle* base_image_handle, struct heif_encoder* encoder,
+    const struct heif_encoding_options* input_options, const uint8_t* gain_map_metadata,
+    int gain_map_metadata_len, const struct heif_color_profile_nclx* derived_image_nclx,
+    struct heif_image_handle** out_image_handle) {
+  if (!encoder) {
+    return Error(heif_error_Usage_error, heif_suberror_Null_pointer_argument)
+        .error_struct(ctx->context.get());
+  }
+
+  if (out_image_handle) {
+    *out_image_handle = nullptr;
+  }
+
+  if (gain_map_metadata_len <= 0) {
+    return Error(heif_error_Invalid_input, heif_suberror_Invalid_parameter_value)
+        .error_struct(ctx->context.get());
+  }
+
+  // --- write tmap item
+  std::vector<uint8_t> metadata;
+  metadata.push_back(0);  // version = 0
+  for (int i = 0; i < gain_map_metadata_len; i++) {
+    metadata.push_back(gain_map_metadata[i]);
+  }
+  heif_item_id tmap_item_id = -1;
+  ctx->context->add_tmap_item(metadata, tmap_item_id);
+
+  std::vector<std::shared_ptr<Box>> properties;
+
+  // --- write ISPE property for tmap item
+  std::shared_ptr<Box_ispe> ispe = std::make_shared<Box_ispe>();
+  ispe->set_size(base_image_handle->image->get_ispe_width(),
+                 base_image_handle->image->get_ispe_height());
+
+  properties.push_back(ispe);
+
+  // --- write PIXI property for tmap item
+  // TODO: this assumes the base image is 8 bit and derived image is 10 bit. Have to handle the
+  // other way where the base image is hdr and derived image is sdr
+  std::shared_ptr<Box_pixi> pixi = std::make_shared<Box_pixi>();
+  pixi->add_channel_bits(10);
+  pixi->add_channel_bits(10);
+  pixi->add_channel_bits(10);
+
+  properties.push_back(pixi);
+
+  // --- write COLR property for tmap item
+  if (derived_image_nclx != nullptr) {
+    std::shared_ptr<Box_colr> colr = std::make_shared<Box_colr>();
+    auto target_nclx_profile = std::make_shared<color_profile_nclx>();
+    target_nclx_profile->set_colour_primaries(derived_image_nclx->color_primaries);
+    target_nclx_profile->set_transfer_characteristics(derived_image_nclx->transfer_characteristics);
+    target_nclx_profile->set_matrix_coefficients(derived_image_nclx->matrix_coefficients);
+    target_nclx_profile->set_full_range_flag(derived_image_nclx->full_range_flag);
+    colr->set_color_profile(target_nclx_profile);
+
+    properties.push_back(colr);
+  }
+  // set item properties
+  for (auto& propertyBox : properties) {
+    int index =
+        ctx->context->get_heif_file()->get_ipco_box()->find_or_append_child_box(propertyBox);
+    ctx->context->get_heif_file()->get_ipma_box()->add_property_for_item_ID(
+        tmap_item_id,
+        Box_ipma::PropertyAssociation{propertyBox->is_essential(), uint16_t(index + 1)});
+  }
+
+  heif_encoding_options options;
+  set_default_encoding_options(options);
+  if (input_options != nullptr) {
+    copy_options(options, *input_options);
+  }
+
+  auto gainmap_encoding_result = ctx->context->encode_image(
+      alternate_image->image, encoder, options, heif_image_input_class_gain_map);
+  if (gainmap_encoding_result.error) {
+    return gainmap_encoding_result.error.error_struct(ctx->context.get());
+  }
+
+  std::shared_ptr<ImageItem> gain_map_image = *gainmap_encoding_result;
+  Error error = ctx->context->link_gain_map(base_image_handle->image, gain_map_image, tmap_item_id);
+  if (error != Error::Ok) {
+    return error.error_struct(ctx->context.get());
+  }
+
+  if (out_image_handle) {
+    *out_image_handle = new heif_image_handle;
+    (*out_image_handle)->image = std::move(gain_map_image);
+    (*out_image_handle)->context = ctx->context;
+  }
+
+  // --- generate altr box
+  auto altr_box = std::make_shared<Box_EntityToGroup>();
+  altr_box->set_short_type(fourcc("altr"));
+  altr_box->set_group_id(ctx->context->get_heif_file()->get_unused_item_id());
+
+  std::vector<heif_item_id> ids;
+  ids.push_back(tmap_item_id);
+  ids.push_back(base_image_handle->image->get_id());
+
+  altr_box->set_item_ids(ids);
+  ctx->context->get_heif_file()->add_entity_group_box(altr_box);
+
+  return heif_error_success;
+}
+#endif
+
+
 struct heif_error heif_context_encode_grid(struct heif_context* ctx,
                                            struct heif_image** tiles,
                                            uint16_t columns,

libheif/api/libheif/heif.h

@@ -884,6 +884,13 @@ typedef uint32_t heif_brand2;
 */
 #define heif_brand2_1pic   heif_fourcc('1','p','i','c')
 
+/**
+ * HEIF tone map brand (`tmap`).
+ *
+ * This is a compatible brand indicating the file contains a gainmap image.
+ */
+#define heif_brand2_tmap heif_fourcc('t', 'm', 'a', 'p')
+
 // input data should be at least 12 bytes
 LIBHEIF_API
 heif_brand2 heif_read_main_brand(const uint8_t* data, int len);
@@ -2621,6 +2628,41 @@ LIBHEIF_API
 int heif_encoder_descriptor_supportes_lossless_compression(const struct heif_encoder_descriptor*);
 
 
+
+// ====================================================================================================
+//  Gain Map API
+
+#if WITH_EXPERIMENTAL_GAIN_MAP
+
+// Get a handle to the gain map image of the HEIF file. If no gain map image is available, this
+// method will return heif_suberror_No_item_data.
+LIBHEIF_API
+struct heif_error heif_context_get_gain_map_image_handle(struct heif_context* ctx,
+                                                         struct heif_image_handle** img);
+
+LIBHEIF_API
+size_t heif_context_get_gain_map_metadata_size(struct heif_context* ctx);
+
+LIBHEIF_API
+struct heif_error heif_context_get_gain_map_metadata(struct heif_context* ctx, void* out_data);
+
+LIBHEIF_API
+struct heif_error heif_context_get_tmap_nclx_color_profile(
+    struct heif_context* ctx, struct heif_color_profile_nclx** out_data);
+
+// Compress the gain map image and write metadata.
+// Returns a handle to the coded image in 'out_image_handle' unless out_image_handle = NULL.
+LIBHEIF_API
+struct heif_error heif_context_encode_gain_map_image(
+    struct heif_context* ctx, const struct heif_image* alternate_image,
+    const struct heif_image_handle* base_image_handle, struct heif_encoder* encoder,
+    const struct heif_encoding_options* input_options, const uint8_t* gain_map_metadata,
+    int gain_map_metadata_len, const struct heif_color_profile_nclx* derived_image_nclx,
+    struct heif_image_handle** out_image_handle);
+
+#endif
+
+
 #ifdef __cplusplus
 }
 #endif

libheif/api/libheif/heif_plugin.h

@@ -129,6 +129,9 @@ enum heif_image_input_class
   heif_image_input_class_alpha = 2,
   heif_image_input_class_depth = 3,
   heif_image_input_class_thumbnail = 4
+#if WITH_EXPERIMENTAL_GAIN_MAP
+  , heif_image_input_class_gain_map = 5
+#endif
 };