feat: offscreen rendering support rgbaf16 hdr output format. (#48504)

feat: offscreen rendering support rgbaf16 hdr output format. (#48265)

* feat: offscreen rendering support rgbaf16

* docs: update doc

* docs: update doc.

Co-authored-by: trop[bot] <37223003+trop[bot]@users.noreply.github.com>
Co-authored-by: reito <reito@chromium.org>

docs/api/structures/offscreen-shared-texture.md

@@ -2,7 +2,10 @@
 
 * `textureInfo` Object - The shared texture info.
   * `widgetType` string - The widget type of the texture. Can be `popup` or `frame`.
-  * `pixelFormat` string - The pixel format of the texture. Can be `rgba` or `bgra`.
+  * `pixelFormat` string - The pixel format of the texture.
+    * `rgba` - The texture format is 8-bit unorm RGBA.
+    * `bgra` - The texture format is 8-bit unorm BGRA.
+    * `rgbaf16` - The texture format is 16-bit float RGBA.
   * `codedSize` [Size](size.md) - The full dimensions of the video frame.
   * `colorSpace` [ColorSpace](color-space.md) - The color space of the video frame.
   * `visibleRect` [Rectangle](rectangle.md) - A subsection of [0, 0, codedSize.width, codedSize.height]. In OSR case, it is expected to have the full section area.

docs/api/structures/web-preferences.md

@@ -87,6 +87,11 @@
      paint event. Defaults to `false`. See the
     [offscreen rendering tutorial](../../tutorial/offscreen-rendering.md) for
     more details.
+  * `sharedTexturePixelFormat` string (optional) _Experimental_ - The requested output format of the shared texture. Defaults to `argb`.
+    The name is originated from Chromium [`media::VideoPixelFormat`](https://source.chromium.org/chromium/chromium/src/+/main:media/base/video_types.h) enum suffix and only subset of them are supported.
+    The actual output pixel format and color space of the texture should refer to [`OffscreenSharedTexture`](../structures/offscreen-shared-texture.md) object in the `paint` event.
+    * `argb` - The requested output texture format is 8-bit unorm RGBA, with SRGB SDR color space.
+    * `rgbaf16` - The requested output texture format is 16-bit float RGBA, with scRGB HDR color space.
 * `contextIsolation` boolean (optional) - Whether to run Electron APIs and
   the specified `preload` script in a separate JavaScript context. Defaults
   to `true`. The context that the `preload` script runs in will only have

shell/browser/api/electron_api_web_contents.cc

@@ -814,6 +814,8 @@ WebContents::WebContents(v8::Isolate* isolate,
       options.Get(options::kOffscreen, &use_offscreen_dict);
       use_offscreen_dict.Get(options::kUseSharedTexture,
                              &offscreen_use_shared_texture_);
+      use_offscreen_dict.Get(options::kSharedTexturePixelFormat,
+                             &offscreen_shared_texture_pixel_format_);
     }
   }
 
@@ -851,6 +853,7 @@ WebContents::WebContents(v8::Isolate* isolate,
     if (embedder_ && embedder_->IsOffScreen()) {
       auto* view = new OffScreenWebContentsView(
           false, offscreen_use_shared_texture_,
+          offscreen_shared_texture_pixel_format_,
           base::BindRepeating(&WebContents::OnPaint, base::Unretained(this)));
       params.view = view;
       params.delegate_view = view;
@@ -872,6 +875,7 @@ WebContents::WebContents(v8::Isolate* isolate,
     content::WebContents::CreateParams params(session->browser_context());
     auto* view = new OffScreenWebContentsView(
         transparent, offscreen_use_shared_texture_,
+        offscreen_shared_texture_pixel_format_,
         base::BindRepeating(&WebContents::OnPaint, base::Unretained(this)));
     params.view = view;
     params.delegate_view = view;
@@ -1229,6 +1233,7 @@ void WebContents::MaybeOverrideCreateParamsForNewWindow(
     if (is_offscreen) {
       auto* view = new OffScreenWebContentsView(
           false, offscreen_use_shared_texture_,
+          offscreen_shared_texture_pixel_format_,
           base::BindRepeating(&WebContents::OnPaint, base::Unretained(this)));
       create_params->view = view;
       create_params->delegate_view = view;

shell/browser/api/electron_api_web_contents.h

@@ -816,6 +816,7 @@ class WebContents final : public ExclusiveAccessContext,
 
   // Whether offscreen rendering use gpu shared texture
   bool offscreen_use_shared_texture_ = false;
+  std::string offscreen_shared_texture_pixel_format_ = "argb";
 
   // Whether window is fullscreened by HTML5 api.
   bool html_fullscreen_ = false;

shell/browser/osr/osr_render_widget_host_view.cc

@@ -154,6 +154,7 @@ class ElectronDelegatedFrameHostClient
 OffScreenRenderWidgetHostView::OffScreenRenderWidgetHostView(
     bool transparent,
     bool offscreen_use_shared_texture,
+    const std::string& offscreen_shared_texture_pixel_format,
     bool painting,
     int frame_rate,
     const OnPaintCallback& callback,
@@ -165,6 +166,8 @@ OffScreenRenderWidgetHostView::OffScreenRenderWidgetHostView(
       parent_host_view_(parent_host_view),
       transparent_(transparent),
       offscreen_use_shared_texture_(offscreen_use_shared_texture),
+      offscreen_shared_texture_pixel_format_(
+          offscreen_shared_texture_pixel_format),
       callback_(callback),
       frame_rate_(frame_rate),
       size_(initial_size),
@@ -548,7 +551,8 @@ OffScreenRenderWidgetHostView::CreateViewForWidget(
   }
 
   return new OffScreenRenderWidgetHostView(
-      transparent_, offscreen_use_shared_texture_, true,
+      transparent_, offscreen_use_shared_texture_,
+      offscreen_shared_texture_pixel_format_, true,
       embedder_host_view->frame_rate(), callback_, render_widget_host,
       embedder_host_view, size());
 }

shell/browser/osr/osr_render_widget_host_view.h

@@ -69,14 +69,16 @@ class OffScreenRenderWidgetHostView
       public ui::CompositorDelegate,
       private OffscreenViewProxyObserver {
  public:
-  OffScreenRenderWidgetHostView(bool transparent,
-                                bool offscreen_use_shared_texture,
-                                bool painting,
-                                int frame_rate,
-                                const OnPaintCallback& callback,
-                                content::RenderWidgetHost* render_widget_host,
-                                OffScreenRenderWidgetHostView* parent_host_view,
-                                gfx::Size initial_size);
+  OffScreenRenderWidgetHostView(
+      bool transparent,
+      bool offscreen_use_shared_texture,
+      const std::string& offscreen_shared_texture_pixel_format,
+      bool painting,
+      int frame_rate,
+      const OnPaintCallback& callback,
+      content::RenderWidgetHost* render_widget_host,
+      OffScreenRenderWidgetHostView* parent_host_view,
+      gfx::Size initial_size);
   ~OffScreenRenderWidgetHostView() override;
 
   // disable copy
@@ -238,6 +240,10 @@ class OffScreenRenderWidgetHostView
     return offscreen_use_shared_texture_;
   }
 
+  const std::string offscreen_shared_texture_pixel_format() const {
+    return offscreen_shared_texture_pixel_format_;
+  }
+
   ui::Layer* root_layer() const { return root_layer_.get(); }
 
   content::DelegatedFrameHost* delegated_frame_host() const {
@@ -283,6 +289,7 @@ class OffScreenRenderWidgetHostView
 
   const bool transparent_;
   const bool offscreen_use_shared_texture_;
+  const std::string offscreen_shared_texture_pixel_format_;
   OnPaintCallback callback_;
   OnPopupPaintCallback parent_callback_;
 

shell/browser/osr/osr_video_consumer.cc

@@ -16,6 +16,22 @@
 #include "third_party/skia/include/core/SkRegion.h"
 #include "ui/gfx/skbitmap_operations.h"
 
+namespace {
+
+media::VideoPixelFormat GetTargetPixelFormatFromOption(
+    const std::string& pixel_format_option) {
+  if (pixel_format_option == "argb") {
+    return media::PIXEL_FORMAT_ARGB;
+  } else if (pixel_format_option == "rgbaf16") {
+    return media::PIXEL_FORMAT_RGBAF16;
+  }
+
+  // Use ARGB as default.
+  return media::PIXEL_FORMAT_ARGB;
+}
+
+}  // namespace
+
 namespace electron {
 
 OffScreenVideoConsumer::OffScreenVideoConsumer(
@@ -26,7 +42,10 @@ OffScreenVideoConsumer::OffScreenVideoConsumer(
       video_capturer_(view->CreateVideoCapturer()) {
   video_capturer_->SetAutoThrottlingEnabled(false);
   video_capturer_->SetMinSizeChangePeriod(base::TimeDelta());
-  video_capturer_->SetFormat(media::PIXEL_FORMAT_ARGB);
+
+  auto format = GetTargetPixelFormatFromOption(
+      view->offscreen_shared_texture_pixel_format());
+  video_capturer_->SetFormat(format);
 
   // https://crrev.org/c/6438681
   // Disable capturer's animation lock-in feature for offscreen capture to

shell/browser/osr/osr_web_contents_view.cc

@@ -16,9 +16,12 @@ namespace electron {
 OffScreenWebContentsView::OffScreenWebContentsView(
     bool transparent,
     bool offscreen_use_shared_texture,
+    const std::string& offscreen_shared_texture_pixel_format,
     const OnPaintCallback& callback)
     : transparent_(transparent),
       offscreen_use_shared_texture_(offscreen_use_shared_texture),
+      offscreen_shared_texture_pixel_format_(
+          offscreen_shared_texture_pixel_format),
       callback_(callback) {
 #if BUILDFLAG(IS_MAC)
   PlatformCreate();
@@ -112,7 +115,8 @@ OffScreenWebContentsView::CreateViewForWidget(
     return static_cast<content::RenderWidgetHostViewBase*>(rwhv);
 
   return new OffScreenRenderWidgetHostView(
-      transparent_, offscreen_use_shared_texture_, painting_, GetFrameRate(),
+      transparent_, offscreen_use_shared_texture_,
+      offscreen_shared_texture_pixel_format_, painting_, GetFrameRate(),
       callback_, render_widget_host, nullptr, GetSize());
 }
 
@@ -132,7 +136,8 @@ OffScreenWebContentsView::CreateViewForChildWidget(
   }
 
   return new OffScreenRenderWidgetHostView(
-      transparent_, offscreen_use_shared_texture_, painting_,
+      transparent_, offscreen_use_shared_texture_,
+      offscreen_shared_texture_pixel_format_, painting_,
       embedder_host_view->frame_rate(), callback_, render_widget_host,
       embedder_host_view, GetSize());
 }

shell/browser/osr/osr_web_contents_view.h

@@ -34,9 +34,11 @@ class OffScreenWebContentsView : public content::WebContentsView,
                                  public content::RenderViewHostDelegateView,
                                  private NativeWindowObserver {
  public:
-  OffScreenWebContentsView(bool transparent,
-                           bool offscreen_use_shared_texture,
-                           const OnPaintCallback& callback);
+  OffScreenWebContentsView(
+      bool transparent,
+      bool offscreen_use_shared_texture,
+      const std::string& offscreen_shared_texture_pixel_format,
+      const OnPaintCallback& callback);
   ~OffScreenWebContentsView() override;
 
   void SetWebContents(content::WebContents*);
@@ -109,6 +111,7 @@ class OffScreenWebContentsView : public content::WebContentsView,
 
   const bool transparent_;
   const bool offscreen_use_shared_texture_;
+  const std::string offscreen_shared_texture_pixel_format_;
   bool painting_ = true;
   int frame_rate_ = 60;
   OnPaintCallback callback_;

shell/common/gin_converters/osr_converter.cc

@@ -29,6 +29,8 @@ std::string OsrVideoPixelFormatToString(media::VideoPixelFormat format) {
       return "bgra";
     case media::PIXEL_FORMAT_ABGR:
       return "rgba";
+    case media::PIXEL_FORMAT_RGBAF16:
+      return "rgbaf16";
     default:
       NOTREACHED();
   }

shell/common/options_switches.h

@@ -183,6 +183,9 @@ inline constexpr std::string_view kOffscreen = "offscreen";
 
 inline constexpr std::string_view kUseSharedTexture = "useSharedTexture";
 
+inline constexpr std::string_view kSharedTexturePixelFormat =
+    "sharedTexturePixelFormat";
+
 inline constexpr std::string_view kNodeIntegrationInSubFrames =
     "nodeIntegrationInSubFrames";