fix(🐛): ImageBitmap alpha and options (#434)

Two snapshot-based suites covering createImageBitmap and
copyExternalImageToTexture, with baselines validated against Chrome
(yarn test:ref) and the dawn.node client (yarn test:node):

Author: wcandillon

apps/example/ios/Podfile.lock

@@ -3074,7 +3074,7 @@ SPEC CHECKSUMS:
   React-microtasksnativemodule: 75b6604b667d297292345302cc5bfb6b6aeccc1b
   react-native-safe-area-context: c00143b4823773bba23f2f19f85663ae89ceb460
   react-native-skia: fc73e9bdc46ebb420a98c9c2be29fee80f565e79
-  react-native-webgpu: 3eb051bebe030f328725fc318a147599c4628e4e
+  react-native-webgpu: cc416064f9c8a68c6fde764a6c36305676f5fd65
   React-NativeModulesApple: 879fbdc5dcff7136abceb7880fe8a2022a1bd7c3
   React-oscompat: 93b5535ea7f7dff46aaee4f78309a70979bdde9d
   React-perflogger: 5536d2df3d18fe0920263466f7b46a56351c0510

apps/example/src/useClient.ts

@@ -1,9 +1,12 @@
 import { useEffect, useState } from "react";
 import { Platform } from "react-native";
 
-const ANDROID_WS_HOST = "10.0.2.2";
-const IOS_WS_HOST = "localhost";
-const HOST = Platform.OS === "android" ? ANDROID_WS_HOST : IOS_WS_HOST;
+// Both platforms reach the test server on localhost: the iOS Simulator shares
+// the host network, and on Android the jest globalSetup runs
+// "adb reverse tcp:4242 tcp:4242" (as the RN CLI does for Metro on 8081), which
+// covers the emulator and a physical device alike. Only a device that cannot be
+// reached over adb needs this machine's LAN IP here instead.
+const HOST = "localhost";
 const PORT = 4242;
 
 type UseClient = [client: WebSocket | null, hostname: string];
@@ -27,7 +30,9 @@ export const useClient = (): UseClient => {
     };
     ws.onopen = () => {
       setClient(ws);
-      ws.send(JSON.stringify({ OS: Platform.OS, arch: "paper" }));
+      // The host is reported back so the test server can hand out fixture URLs
+      // this device can actually reach (see fixtureUrl in setup.ts).
+      ws.send(JSON.stringify({ OS: Platform.OS, arch: "paper", host: HOST }));
     };
     // Reconnect on every close, not only on error: the test server closes the
     // socket cleanly at the end of each jest run, and without a retry here the

packages/webgpu/android/cpp/AndroidPlatformContext.h

@@ -179,6 +179,9 @@ class AndroidPlatformContext : public PlatformContext {
     result.height = static_cast<int>(bitmapInfo.height);
     result.data.resize(bitmapInfo.height * bitmapInfo.stride);
     memcpy(result.data.data(), bitmapPixels, result.data.size());
+    // BitmapFactory hands back premultiplied ARGB_8888 pixels; createImageBitmap
+    // converts to the representation requested by premultiplyAlpha.
+    result.premultiplied = true;
 
     AndroidBitmap_unlockPixels(env, bitmap);
 

packages/webgpu/apple/ApplePlatformContext.mm

@@ -4,6 +4,7 @@
 
 #import <AVFoundation/AVFoundation.h>
 #import <CoreVideo/CoreVideo.h>
+#import <ImageIO/ImageIO.h>
 #import <React/RCTBlobManager.h>
 #import <React/RCTBridge+Private.h>
 #import <ReactCommon/RCTTurboModule.h>
@@ -88,36 +89,32 @@ void checkIfUsingSimulatorWithAPIValidation() {
 
 ImageData
 ApplePlatformContext::createImageBitmapFromData(std::span<const uint8_t> data) {
-  // This avoids a copy by assuming the UIImage/NSImage constructors
-  // decode `nsData` eagerly before the memory for the wrapped `data`
-  // is freed.
-  //
-  // Since we get the `CGImageRef` from `image` and then throw
-  // it away, that's a fairly safe assumption.
+  // All formats are decoded through ImageIO. Apple's imaging stack always
+  // premultiplies alpha at decode, so the result is flagged premultiplied and
+  // createImageBitmap / copyExternalImageToTexture convert from there. This
+  // means premultiplyAlpha "none" is a lossy round trip for low-alpha pixels
+  // (as it is on Android); the snapshot suites compare with pixelmatch
+  // tolerance to absorb it.
   NSData *nsData =
       [NSData dataWithBytesNoCopy:const_cast<uint8_t *>(data.data())
                            length:data.size()
                      freeWhenDone:NO];
 
-#if !TARGET_OS_OSX
-  UIImage *image = [UIImage imageWithData:nsData];
-#else
-  NSImage *image = [[NSImage alloc] initWithData:nsData];
-#endif
-  if (!image) {
+  CGImageSourceRef imageSource =
+      CGImageSourceCreateWithData((__bridge CFDataRef)nsData, NULL);
+  if (imageSource == NULL) {
+    throw std::runtime_error("Couldn't create image source");
+  }
+  NSDictionary *decodeOptions = @{(id)kCGImageSourceShouldCache : @NO};
+  CGImageRef cgImage = CGImageSourceCreateImageAtIndex(
+      imageSource, 0, (__bridge CFDictionaryRef)decodeOptions);
+  CFRelease(imageSource);
+  if (cgImage == NULL) {
     throw std::runtime_error("Couldn't decode image");
   }
 
-#if !TARGET_OS_OSX
-  CGImageRef cgImage = image.CGImage;
-#else
-  CGImageRef cgImage = [image CGImageForProposedRect:NULL
-                                             context:NULL
-                                               hints:NULL];
-#endif
   size_t width = CGImageGetWidth(cgImage);
   size_t height = CGImageGetHeight(cgImage);
-  size_t bitsPerComponent = 8;
   size_t bytesPerRow = width * 4;
 
   ImageData result;
@@ -126,16 +123,18 @@ void checkIfUsingSimulatorWithAPIValidation() {
   result.data.resize(height * bytesPerRow);
   result.format = wgpu::TextureFormat::RGBA8Unorm;
 
+  // The draw premultiplies alpha; flag the result premultiplied so
+  // createImageBitmap and copyExternalImageToTexture convert consistently.
   CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
   CGContextRef context = CGBitmapContextCreate(
-      result.data.data(), width, height, bitsPerComponent, bytesPerRow,
-      colorSpace, kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big);
-
+      result.data.data(), width, height, 8, bytesPerRow, colorSpace,
+      kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big);
   CGContextDrawImage(context, CGRectMake(0, 0, width, height), cgImage);
-
   CGContextRelease(context);
   CGColorSpaceRelease(colorSpace);
+  CGImageRelease(cgImage);
 
+  result.premultiplied = true;
   return result;
 }
 

packages/webgpu/cpp/rnwgpu/PlatformContext.h

@@ -16,6 +16,13 @@ struct ImageData {
   size_t width;
   size_t height;
   wgpu::TextureFormat format;
+  // Whether the RGB channels are premultiplied by alpha. Each platform decoder
+  // records the representation it produced; createImageBitmap then converts to
+  // the representation requested through premultiplyAlpha, and
+  // copyExternalImageToTexture converts again to match the destination's
+  // premultipliedAlpha. Defaults to true because most native decoders
+  // (CoreGraphics, Android Bitmap) hand back premultiplied pixels.
+  bool premultiplied = true;
 };
 
 // Pixel layout of a VideoFrame. Determines whether the underlying surface is

packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp

@@ -128,26 +128,40 @@ void GPUQueue::copyExternalImageToTexture(
     throw std::runtime_error("Invalid input for GPUQueue::writeTexture()");
   }
 
-  if (source->flipY.value_or(false)) {
-    // Calculate the row size and total size
+  const bool flipY = source->flipY.value_or(false);
+  // premultipliedAlpha defaults to false per the WebGPU spec: an untagged
+  // destination expects straight alpha. Convert only when the ImageBitmap's
+  // representation differs, using the same rounding as the reference client.
+  const bool sourcePremultiplied = source->source->getPremultiplied();
+  const bool destinationPremultiplied =
+      destination->premultipliedAlpha.value_or(false);
+  const bool needsAlphaConversion =
+      bytesPerPixel == 4 && sourcePremultiplied != destinationPremultiplied;
+
+  if (flipY || needsAlphaConversion) {
     uint32_t rowSize = bytesPerPixel * source->source->getWidth();
     uint32_t totalSize = source->source->getSize();
 
-    // Create a new buffer for the flipped data
-    std::vector<uint8_t> flippedData(totalSize);
-
-    // Flip the data vertically
-    for (uint32_t row = 0; row < source->source->getHeight(); ++row) {
-      std::memcpy(flippedData.data() +
-                      (source->source->getHeight() - 1 - row) * rowSize,
-                  static_cast<const uint8_t *>(source->source->getData()) +
-                      row * rowSize,
-                  rowSize);
+    // Stage a mutable copy so flipping and/or alpha conversion never touch the
+    // ImageBitmap's backing store.
+    std::vector<uint8_t> staged(totalSize);
+    const uint8_t *src =
+        static_cast<const uint8_t *>(source->source->getData());
+    if (flipY) {
+      for (uint32_t row = 0; row < source->source->getHeight(); ++row) {
+        std::memcpy(staged.data() +
+                        (source->source->getHeight() - 1 - row) * rowSize,
+                    src + row * rowSize, rowSize);
+      }
+    } else {
+      std::memcpy(staged.data(), src, totalSize);
+    }
+    if (needsAlphaConversion) {
+      convertAlpha(staged.data(), totalSize, sourcePremultiplied,
+                   destinationPremultiplied);
     }
-    // Use the flipped data for writing to texture
-    _instance.WriteTexture(&dst, flippedData.data(), totalSize, &layout, &sz);
+    _instance.WriteTexture(&dst, staged.data(), totalSize, &layout, &sz);
   } else {
-
     _instance.WriteTexture(&dst, source->source->getData(),
                            source->source->getSize(), &layout, &sz);
   }

packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h

@@ -11,6 +11,33 @@ namespace rnwgpu {
 
 namespace jsi = facebook::jsi;
 
+// Convert RGBA8 pixel data in place between straight and premultiplied alpha.
+// Uses the same integer rounding as the web reference polyfill in the test
+// harness (setup.ts, convertAlpha) so the native result is bit-exact with the
+// dawn.node client. A no-op when the source and destination representations
+// already match.
+inline void convertAlpha(uint8_t *data, size_t byteLength,
+                         bool sourcePremultiplied,
+                         bool destinationPremultiplied) {
+  if (sourcePremultiplied == destinationPremultiplied) {
+    return;
+  }
+  for (size_t i = 0; i + 3 < byteLength; i += 4) {
+    const uint32_t alpha = data[i + 3];
+    for (size_t channel = 0; channel < 3; channel++) {
+      const uint32_t value = data[i + channel];
+      if (destinationPremultiplied) {
+        data[i + channel] = static_cast<uint8_t>((value * alpha + 127) / 255);
+      } else if (alpha == 0) {
+        data[i + channel] = 0;
+      } else {
+        const uint32_t straight = (value * 255 + (alpha >> 1)) / alpha;
+        data[i + channel] = static_cast<uint8_t>(straight > 255 ? 255 : straight);
+      }
+    }
+  }
+}
+
 class ImageBitmap : public NativeObject<ImageBitmap> {
 public:
   static constexpr const char *CLASS_NAME = "ImageBitmap";
@@ -26,6 +53,11 @@ class ImageBitmap : public NativeObject<ImageBitmap> {
 
   size_t getSize() { return _imageData.data.size(); }
 
+  // Whether the stored pixels are premultiplied by alpha. Used by
+  // copyExternalImageToTexture to decide whether a conversion to the
+  // destination's premultipliedAlpha representation is needed.
+  bool getPremultiplied() { return _imageData.premultiplied; }
+
   void close() {
     _imageData.data.clear();
     _imageData.data.shrink_to_fit();

packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h

@@ -90,6 +90,41 @@ class RNWebGPU : public NativeObject<RNWebGPU> {
     auto platformContext = _platformContext;
     auto callInvoker = _callInvoker;
 
+    // Resolve the requested alpha representation from the ImageBitmapOptions.
+    // The options bag is the second argument for createImageBitmap(source,
+    // options) and the sixth for the crop-rect overload createImageBitmap(
+    // source, sx, sy, sw, sh, options). premultiplyAlpha defaults to
+    // "default", which (like "premultiply") stores premultiplied pixels; only
+    // "none" keeps straight alpha. The other options (crop rect, resize,
+    // imageOrientation, colorSpaceConversion) are not yet implemented natively.
+    bool wantPremultiplied = true;
+    const jsi::Value *optionsArg = nullptr;
+    if (count >= 2 && args[1].isObject()) {
+      optionsArg = &args[1];
+    } else if (count >= 6 && args[5].isObject()) {
+      optionsArg = &args[5];
+    }
+    if (optionsArg != nullptr) {
+      auto options = optionsArg->getObject(runtime);
+      if (options.hasProperty(runtime, "premultiplyAlpha")) {
+        auto value = options.getProperty(runtime, "premultiplyAlpha");
+        if (value.isString() &&
+            value.getString(runtime).utf8(runtime) == "none") {
+          wantPremultiplied = false;
+        }
+      }
+    }
+
+    // Bring the decoded pixels into the representation requested via
+    // premultiplyAlpha before wrapping them in an ImageBitmap.
+    auto toRequestedAlpha = [wantPremultiplied](ImageData &imageData) {
+      if (imageData.premultiplied != wantPremultiplied) {
+        convertAlpha(imageData.data.data(), imageData.data.size(),
+                     imageData.premultiplied, wantPremultiplied);
+        imageData.premultiplied = wantPremultiplied;
+      }
+    };
+
     // Check if the argument is an ArrayBuffer or ArrayBufferView
     // (TypedArray / DataView). Only a real buffer source is run through the
     // ArrayBuffer converter, which validates byteOffset/byteLength against the
@@ -119,12 +154,14 @@ class RNWebGPU : public NativeObject<RNWebGPU> {
 
         return Promise::createPromise(
             runtime,
-            [platformContext, callInvoker, dataCopy = std::move(dataCopy)](
+            [platformContext, callInvoker, toRequestedAlpha,
+             dataCopy = std::move(dataCopy)](
                 jsi::Runtime & /*runtime*/,
                 std::shared_ptr<Promise> promise) mutable {
               platformContext->createImageBitmapFromDataAsync(
                   dataCopy,
-                  [callInvoker, promise](ImageData imageData) {
+                  [callInvoker, promise, toRequestedAlpha](ImageData imageData) {
+                    toRequestedAlpha(imageData);
                     auto imageBitmap = std::make_shared<ImageBitmap>(imageData);
                     callInvoker->invokeAsync([promise, imageBitmap]() {
                       promise->resolve(
@@ -149,11 +186,12 @@ class RNWebGPU : public NativeObject<RNWebGPU> {
 
     return Promise::createPromise(
         runtime,
-        [platformContext, callInvoker, blobId, offset,
+        [platformContext, callInvoker, toRequestedAlpha, blobId, offset,
          size](jsi::Runtime & /*runtime*/, std::shared_ptr<Promise> promise) {
           platformContext->createImageBitmapAsync(
               blobId, offset, size,
-              [callInvoker, promise](ImageData imageData) {
+              [callInvoker, promise, toRequestedAlpha](ImageData imageData) {
+                toRequestedAlpha(imageData);
                 auto imageBitmap = std::make_shared<ImageBitmap>(imageData);
                 callInvoker->invokeAsync([promise, imageBitmap]() {
                   promise->resolve(

packages/webgpu/react-native-webgpu.podspec

@@ -23,7 +23,8 @@ Pod::Spec.new do |s|
 
   # The VideoPlayer API uses AVFoundation / CoreMedia, and shared-texture
   # surfaces use CoreVideo (CVPixelBuffer). Link them so their symbols resolve.
-  s.frameworks = "AVFoundation", "CoreMedia", "CoreVideo"
+  # ImageIO provides CGImageSource, the image decoder behind createImageBitmap.
+  s.frameworks = "AVFoundation", "CoreMedia", "CoreVideo", "ImageIO"
 
   s.pod_target_xcconfig = {
     'HEADER_SEARCH_PATHS' => '$(PODS_TARGET_SRCROOT)/cpp',