From 9dcdec5015bfae8d57140d0f4b25ed1cdd9f91ff Mon Sep 17 00:00:00 2001 From: reito Date: Fri, 8 Aug 2025 16:13:56 +0800 Subject: [PATCH] feat: `paint` event move texture data to `handle`, add `colorSpace` (#47315) * feat: paint event move shared texture to handle, add color space * feat: add breaking change * fix: properties. * fix: remove utf8 bom --- .../structures/offscreen-shared-texture.md | 14 ++----- docs/api/structures/shared-texture-handle.md | 12 ++++++ docs/breaking-changes.md | 6 +++ shell/browser/osr/osr_paint_event.h | 4 ++ shell/browser/osr/osr_video_consumer.cc | 3 ++ shell/common/gin_converters/osr_converter.cc | 41 +++++++++++++------ shell/common/gin_converters/osr_converter.h | 7 ++-- 7 files changed, 60 insertions(+), 27 deletions(-) create mode 100644 docs/api/structures/shared-texture-handle.md diff --git a/docs/api/structures/offscreen-shared-texture.md b/docs/api/structures/offscreen-shared-texture.md index 0de21db13397..b930bc5daf25 100644 --- a/docs/api/structures/offscreen-shared-texture.md +++ b/docs/api/structures/offscreen-shared-texture.md @@ -4,7 +4,8 @@ * `widgetType` string - The widget type of the texture. Can be `popup` or `frame`. * `pixelFormat` string - The pixel format of the texture. Can be `rgba` or `bgra`. * `codedSize` [Size](size.md) - The full dimensions of the video frame. - * `visibleRect` [Rectangle](rectangle.md) - A subsection of [0, 0, codedSize.width(), codedSize.height()]. In OSR case, it is expected to have the full section area. + * `colorSpace` [ColorSpace](color-space.md) - The color space of the video frame. + * `visibleRect` [Rectangle](rectangle.md) - A subsection of [0, 0, codedSize.width, codedSize.height]. In OSR case, it is expected to have the full section area. * `contentRect` [Rectangle](rectangle.md) - The region of the video frame that capturer would like to populate. In OSR case, it is the same with `dirtyRect` that needs to be painted. * `timestamp` number - The time in microseconds since the capture start. * `metadata` Object - Extra metadata. See comments in src\media\base\video_frame_metadata.h for accurate details. @@ -12,13 +13,6 @@ * `regionCaptureRect` [Rectangle](rectangle.md) (optional) - May reflect the frame's contents origin if region capture is used internally. * `sourceSize` [Rectangle](rectangle.md) (optional) - Full size of the source frame. * `frameCount` number (optional) - The increasing count of captured frame. May contain gaps if frames are dropped between two consecutively received frames. - * `sharedTextureHandle` Buffer _Windows_ _macOS_ - The handle to the shared texture. - * `planes` Object[] _Linux_ - Each plane's info of the shared texture. - * `stride` number - The strides and offsets in bytes to be used when accessing the buffers via a memory mapping. One per plane per entry. - * `offset` number - The strides and offsets in bytes to be used when accessing the buffers via a memory mapping. One per plane per entry. - * `size` number - Size in bytes of the plane. This is necessary to map the buffers. - * `fd` number - File descriptor for the underlying memory object (usually dmabuf). - * `modifier` string _Linux_ - The modifier is retrieved from GBM library and passed to EGL driver. + * `handle` [SharedTextureHandle](shared-texture-handle.md) - The shared texture handle data. * `release` Function - Release the resources. The `texture` cannot be directly passed to another process, users need to maintain texture lifecycles in - main process, but it is safe to pass the `textureInfo` to another process. Only a limited number of textures can exist at the same time, so it's important - that you call `texture.release()` as soon as you're done with the texture. + main process, but it is safe to pass the `textureInfo` to another process. Only a limited number of textures can exist at the same time, so it's important that you call `texture.release()` as soon as you're done with the texture. diff --git a/docs/api/structures/shared-texture-handle.md b/docs/api/structures/shared-texture-handle.md new file mode 100644 index 000000000000..04d338efe602 --- /dev/null +++ b/docs/api/structures/shared-texture-handle.md @@ -0,0 +1,12 @@ +# SharedTextureHandle Object + +* `ntHandle` Buffer (optional) _Windows_ - NT HANDLE holds the shared texture. Note that this NT HANDLE is local to current process. +* `ioSurface` Buffer (optional) _macOS_ - IOSurfaceRef holds the shared texture. Note that this IOSurface is local to current process (not global). +* `nativePixmap` Object (optional) _Linux_ - Structure contains planes of shared texture. + * `planes` Object[] _Linux_ - Each plane's info of the shared texture. + * `stride` number - The strides and offsets in bytes to be used when accessing the buffers via a memory mapping. One per plane per entry. + * `offset` number - The strides and offsets in bytes to be used when accessing the buffers via a memory mapping. One per plane per entry. + * `size` number - Size in bytes of the plane. This is necessary to map the buffers. + * `fd` number - File descriptor for the underlying memory object (usually dmabuf). + * `modifier` string _Linux_ - The modifier is retrieved from GBM library and passed to EGL driver. + * `supportsZeroCopyWebGpuImport` boolean _Linux_ - Indicates whether supports zero copy import to WebGPU. diff --git a/docs/breaking-changes.md b/docs/breaking-changes.md index ca3572863e43..d9006c70b40f 100644 --- a/docs/breaking-changes.md +++ b/docs/breaking-changes.md @@ -54,6 +54,12 @@ webContents.setWindowOpenHandler((details) => { }) ``` +### Behavior Changed: shared texture OSR `paint` event data structure + +When using shared texture offscreen rendering feature, the `paint` event now emits a more structured object. +It moves the `sharedTextureHandle`, `planes`, `modifier` into a unified `handle` property. +See [here](https://www.electronjs.org/docs/latest/api/structures/offscreen-shared-texture) for more details. + ## Planned Breaking API Changes (37.0) ### Utility Process unhandled rejection behavior change diff --git a/shell/browser/osr/osr_paint_event.h b/shell/browser/osr/osr_paint_event.h index 70985418fc66..cd8f8710c670 100644 --- a/shell/browser/osr/osr_paint_event.h +++ b/shell/browser/osr/osr_paint_event.h @@ -77,6 +77,9 @@ struct OffscreenSharedTextureValue { // In OSR case, it is the same with `dirtyRect` that needs to be painted. gfx::Rect content_rect; + // Color space of the video frame. + gfx::ColorSpace color_space; + // Extra metadata for the video frame. // See comments in src\media\base\video_frame_metadata.h for more details. std::optional capture_update_rect; @@ -99,6 +102,7 @@ struct OffscreenSharedTextureValue { #elif BUILDFLAG(IS_LINUX) std::vector planes; uint64_t modifier; + bool supports_zero_copy_webgpu_import; #endif }; diff --git a/shell/browser/osr/osr_video_consumer.cc b/shell/browser/osr/osr_video_consumer.cc index 00f80701072f..ac00b63a817e 100644 --- a/shell/browser/osr/osr_video_consumer.cc +++ b/shell/browser/osr/osr_video_consumer.cc @@ -95,6 +95,7 @@ void OffScreenVideoConsumer::OnFrameCaptured( texture.coded_size = info->coded_size; texture.visible_rect = info->visible_rect; texture.content_rect = content_rect; + texture.color_space = info->color_space; texture.timestamp = info->timestamp.InMicroseconds(); texture.frame_count = info->metadata.capture_counter.value_or(0); texture.capture_update_rect = info->metadata.capture_update_rect; @@ -111,6 +112,8 @@ void OffScreenVideoConsumer::OnFrameCaptured( #elif BUILDFLAG(IS_LINUX) const auto& native_pixmap = gmb_handle.native_pixmap_handle(); texture.modifier = native_pixmap.modifier; + texture.supports_zero_copy_webgpu_import = + native_pixmap.supports_zero_copy_webgpu_import; for (const auto& plane : native_pixmap.planes) { texture.planes.emplace_back(plane.stride, plane.offset, plane.size, plane.fd.get()); diff --git a/shell/common/gin_converters/osr_converter.cc b/shell/common/gin_converters/osr_converter.cc index 1d636bd24fba..55b94f855c42 100644 --- a/shell/common/gin_converters/osr_converter.cc +++ b/shell/common/gin_converters/osr_converter.cc @@ -1,7 +1,6 @@ - -// Copyright (c) 2024 GitHub, Inc. -// Use of this source code is governed by the MIT license that can be found in -// the LICENSE file. +// Copyright (c) 2025 GitHub, Inc. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. #include "shell/common/gin_converters/osr_converter.h" @@ -17,6 +16,7 @@ #endif #include "shell/common/gin_converters/gfx_converter.h" #include "shell/common/gin_converters/optional_converter.h" +#include "shell/common/gin_helper/error_thrower.h" #include "shell/common/node_includes.h" #include "shell/common/node_util.h" @@ -87,10 +87,11 @@ v8::Local Converter::ToV8( auto releaserHolder = v8::External::New(isolate, monitor); auto releaserFunc = [](const v8::FunctionCallbackInfo& info) { - auto* holder = static_cast( + auto* mon = static_cast( info.Data().As()->Value()); // Release the shared texture, so that future frames can be generated. - holder->ReleaseTexture(); + mon->ReleaseTexture(); + // Release the monitor happens at GC, don't release here. }; auto releaser = v8::Function::New(isolate->GetCurrentContext(), releaserFunc, releaserHolder) @@ -104,6 +105,7 @@ v8::Local Converter::ToV8( dict.Set("visibleRect", val.visible_rect); dict.Set("contentRect", val.content_rect); dict.Set("timestamp", val.timestamp); + dict.Set("colorSpace", val.color_space); dict.Set("widgetType", OsrWidgetTypeToString(val.widget_type)); gin::Dictionary metadata(isolate, v8::Object::New(isolate)); @@ -113,12 +115,21 @@ v8::Local Converter::ToV8( metadata.Set("frameCount", val.frame_count); dict.Set("metadata", ConvertToV8(isolate, metadata)); -#if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC) - dict.Set("sharedTextureHandle", - electron::Buffer::Copy( - isolate, base::byte_span_from_ref(val.shared_texture_handle)) - .ToLocalChecked()); + gin::Dictionary sharedTexture(isolate, v8::Object::New(isolate)); +#if BUILDFLAG(IS_WIN) + sharedTexture.Set( + "ntHandle", + electron::Buffer::Copy( + isolate, base::byte_span_from_ref(val.shared_texture_handle)) + .ToLocalChecked()); +#elif BUILDFLAG(IS_MAC) + sharedTexture.Set( + "ioSurface", + electron::Buffer::Copy( + isolate, base::byte_span_from_ref(val.shared_texture_handle)) + .ToLocalChecked()); #elif BUILDFLAG(IS_LINUX) + gin::Dictionary nativePixmap(isolate, v8::Object::New(isolate)); auto v8_planes = base::ToVector(val.planes, [isolate](const auto& plane) { gin::Dictionary v8_plane(isolate, v8::Object::New(isolate)); v8_plane.Set("stride", plane.stride); @@ -127,10 +138,14 @@ v8::Local Converter::ToV8( v8_plane.Set("fd", plane.fd); return v8_plane; }); - dict.Set("planes", v8_planes); - dict.Set("modifier", base::NumberToString(val.modifier)); + nativePixmap.Set("planes", v8_planes); + nativePixmap.Set("modifier", base::NumberToString(val.modifier)); + nativePixmap.Set("supportsZeroCopyWebGpuImport", + val.supports_zero_copy_webgpu_import); + sharedTexture.Set("nativePixmap", ConvertToV8(isolate, nativePixmap)); #endif + dict.Set("handle", ConvertToV8(isolate, sharedTexture)); root.Set("textureInfo", ConvertToV8(isolate, dict)); auto root_local = ConvertToV8(isolate, root); diff --git a/shell/common/gin_converters/osr_converter.h b/shell/common/gin_converters/osr_converter.h index 05e767f20ae1..d23d9aa4816d 100644 --- a/shell/common/gin_converters/osr_converter.h +++ b/shell/common/gin_converters/osr_converter.h @@ -1,7 +1,6 @@ - -// Copyright (c) 2024 GitHub, Inc. -// Use of this source code is governed by the MIT license that can be found in -// the LICENSE file. +// Copyright (c) 2025 GitHub, Inc. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. #ifndef ELECTRON_SHELL_COMMON_GIN_CONVERTERS_OSR_CONVERTER_H_ #define ELECTRON_SHELL_COMMON_GIN_CONVERTERS_OSR_CONVERTER_H_