From 55c57808fb9105dfc077f612b2d6c7303e0a2522 Mon Sep 17 00:00:00 2001 From: Jeremy Rose Date: Tue, 7 Sep 2021 10:37:45 -0700 Subject: [PATCH] feat: serialize NativeImage over ipc (#30729) --- filenames.auto.gni | 5 - lib/browser/guest-view-manager.ts | 7 -- lib/browser/rpc-server.ts | 5 +- lib/common/api/clipboard.ts | 10 +- lib/common/ipc-messages.ts | 1 - lib/common/type-utils.ts | 110 ------------------- lib/common/web-view-methods.ts | 1 + lib/renderer/api/desktop-capturer.ts | 3 +- lib/renderer/web-view/guest-view-internal.ts | 4 - lib/renderer/web-view/web-view-impl.ts | 6 - shell/common/api/electron_api_native_image.h | 13 +-- shell/common/v8_value_serializer.cc | 69 +++++++++++- shell/renderer/renderer_client_base.cc | 1 - typings/internal-ambient.d.ts | 1 - 14 files changed, 79 insertions(+), 157 deletions(-) delete mode 100644 lib/common/type-utils.ts diff --git a/filenames.auto.gni b/filenames.auto.gni index 92bcbf4f0e7..f2f8f9a6bd6 100644 --- a/filenames.auto.gni +++ b/filenames.auto.gni @@ -138,7 +138,6 @@ auto_filenames = { "lib/common/api/native-image.ts", "lib/common/define-properties.ts", "lib/common/ipc-messages.ts", - "lib/common/type-utils.ts", "lib/common/web-view-methods.ts", "lib/renderer/api/context-bridge.ts", "lib/renderer/api/crash-reporter.ts", @@ -168,7 +167,6 @@ auto_filenames = { ] isolated_bundle_deps = [ - "lib/common/type-utils.ts", "lib/common/web-view-methods.ts", "lib/isolated_renderer/init.ts", "lib/renderer/web-view/web-view-attributes.ts", @@ -246,7 +244,6 @@ auto_filenames = { "lib/common/ipc-messages.ts", "lib/common/parse-features-string.ts", "lib/common/reset-search-paths.ts", - "lib/common/type-utils.ts", "lib/common/web-view-events.ts", "lib/common/web-view-methods.ts", "lib/common/webpack-globals-provider.ts", @@ -269,7 +266,6 @@ auto_filenames = { "lib/common/init.ts", "lib/common/ipc-messages.ts", "lib/common/reset-search-paths.ts", - "lib/common/type-utils.ts", "lib/common/web-view-methods.ts", "lib/common/webpack-provider.ts", "lib/renderer/api/context-bridge.ts", @@ -309,7 +305,6 @@ auto_filenames = { "lib/common/init.ts", "lib/common/ipc-messages.ts", "lib/common/reset-search-paths.ts", - "lib/common/type-utils.ts", "lib/common/webpack-provider.ts", "lib/renderer/api/context-bridge.ts", "lib/renderer/api/crash-reporter.ts", diff --git a/lib/browser/guest-view-manager.ts b/lib/browser/guest-view-manager.ts index dec9ddd4011..59e9c9f87a0 100644 --- a/lib/browser/guest-view-manager.ts +++ b/lib/browser/guest-view-manager.ts @@ -4,7 +4,6 @@ import * as ipcMainUtils from '@electron/internal/browser/ipc-main-internal-util import { parseWebViewWebPreferences } from '@electron/internal/common/parse-features-string'; import { syncMethods, asyncMethods, properties } from '@electron/internal/common/web-view-methods'; import { webViewEvents } from '@electron/internal/common/web-view-events'; -import { serialize } from '@electron/internal/common/type-utils'; import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages'; interface GuestInstance { @@ -330,12 +329,6 @@ handleMessageSync(IPC_MESSAGES.GUEST_VIEW_MANAGER_PROPERTY_SET, function (event, (guest as any)[property] = val; }); -handleMessage(IPC_MESSAGES.GUEST_VIEW_MANAGER_CAPTURE_PAGE, async function (event, guestInstanceId: number, args: any[]) { - const guest = getGuestForWebContents(guestInstanceId, event.sender); - - return serialize(await guest.capturePage(...args)); -}); - // Returns WebContents from its guest id hosted in given webContents. const getGuestForWebContents = function (guestInstanceId: number, contents: Electron.WebContents) { const guestInstance = guestInstances.get(guestInstanceId); diff --git a/lib/browser/rpc-server.ts b/lib/browser/rpc-server.ts index 24da9056634..e67b76b3c99 100644 --- a/lib/browser/rpc-server.ts +++ b/lib/browser/rpc-server.ts @@ -4,7 +4,6 @@ import { clipboard } from 'electron/common'; import * as fs from 'fs'; import { ipcMainInternal } from '@electron/internal/browser/ipc-main-internal'; import * as ipcMainUtils from '@electron/internal/browser/ipc-main-internal-utils'; -import * as typeUtils from '@electron/internal/common/type-utils'; import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages'; import type * as desktopCapturerModule from '@electron/internal/browser/desktop-capturer'; @@ -56,7 +55,7 @@ ipcMainUtils.handleSync(IPC_MESSAGES.BROWSER_CLIPBOARD_SYNC, function (event, me throw new Error(`Invalid method: ${method}`); } - return typeUtils.serialize((clipboard as any)[method](...typeUtils.deserialize(args))); + return (clipboard as any)[method](...args); }); if (BUILDFLAG(ENABLE_DESKTOP_CAPTURER)) { @@ -71,7 +70,7 @@ if (BUILDFLAG(ENABLE_DESKTOP_CAPTURER)) { return []; } - return typeUtils.serialize(await desktopCapturer.getSourcesImpl(event.sender, options)); + return await desktopCapturer.getSourcesImpl(event.sender, options); }); } diff --git a/lib/common/api/clipboard.ts b/lib/common/api/clipboard.ts index e20ef57a943..dd03b4f279d 100644 --- a/lib/common/api/clipboard.ts +++ b/lib/common/api/clipboard.ts @@ -1,20 +1,14 @@ import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages'; import type * as ipcRendererUtilsModule from '@electron/internal/renderer/ipc-renderer-internal-utils'; -import type * as typeUtilsModule from '@electron/internal/common/type-utils'; const clipboard = process._linkedBinding('electron_common_clipboard'); if (process.type === 'renderer') { const ipcRendererUtils = require('@electron/internal/renderer/ipc-renderer-internal-utils') as typeof ipcRendererUtilsModule; - const typeUtils = require('@electron/internal/common/type-utils') as typeof typeUtilsModule; - const makeRemoteMethod = function (method: keyof Electron.Clipboard) { - return (...args: any[]) => { - args = typeUtils.serialize(args); - const result = ipcRendererUtils.invokeSync(IPC_MESSAGES.BROWSER_CLIPBOARD_SYNC, method, ...args); - return typeUtils.deserialize(result); - }; + const makeRemoteMethod = function (method: keyof Electron.Clipboard): any { + return (...args: any[]) => ipcRendererUtils.invokeSync(IPC_MESSAGES.BROWSER_CLIPBOARD_SYNC, method, ...args); }; if (process.platform === 'linux') { diff --git a/lib/common/ipc-messages.ts b/lib/common/ipc-messages.ts index 67fcec2597d..48432d12987 100644 --- a/lib/common/ipc-messages.ts +++ b/lib/common/ipc-messages.ts @@ -13,7 +13,6 @@ export const enum IPC_MESSAGES { GUEST_VIEW_MANAGER_DETACH_GUEST = 'GUEST_VIEW_MANAGER_DETACH_GUEST', GUEST_VIEW_MANAGER_FOCUS_CHANGE = 'GUEST_VIEW_MANAGER_FOCUS_CHANGE', GUEST_VIEW_MANAGER_CALL = 'GUEST_VIEW_MANAGER_CALL', - GUEST_VIEW_MANAGER_CAPTURE_PAGE = 'GUEST_VIEW_MANAGER_CAPTURE_PAGE', GUEST_VIEW_MANAGER_PROPERTY_GET = 'GUEST_VIEW_MANAGER_PROPERTY_GET', GUEST_VIEW_MANAGER_PROPERTY_SET = 'GUEST_VIEW_MANAGER_PROPERTY_SET', diff --git a/lib/common/type-utils.ts b/lib/common/type-utils.ts deleted file mode 100644 index 59b1705b3e1..00000000000 --- a/lib/common/type-utils.ts +++ /dev/null @@ -1,110 +0,0 @@ -function getCreateNativeImage () { - return process._linkedBinding('electron_common_native_image').nativeImage.createEmpty; -} - -export function isPromise (val: any) { - return ( - val && - val.then && - val.then instanceof Function && - val.constructor && - val.constructor.reject && - val.constructor.reject instanceof Function && - val.constructor.resolve && - val.constructor.resolve instanceof Function - ); -} - -const serializableTypes = [ - Boolean, - Number, - String, - Date, - Error, - RegExp, - ArrayBuffer -]; - -// https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm#Supported_types -export function isSerializableObject (value: any) { - return value === null || ArrayBuffer.isView(value) || serializableTypes.some(type => value instanceof type); -} - -const objectMap = function (source: Object, mapper: (value: any) => any) { - const sourceEntries = Object.entries(source); - const targetEntries = sourceEntries.map(([key, val]) => [key, mapper(val)]); - return Object.fromEntries(targetEntries); -}; - -function serializeNativeImage (image: Electron.NativeImage) { - const representations = []; - const scaleFactors = image.getScaleFactors(); - - // Use Buffer when there's only one representation for better perf. - // This avoids compressing to/from PNG where it's not necessary to - // ensure uniqueness of dataURLs (since there's only one). - if (scaleFactors.length === 1) { - const scaleFactor = scaleFactors[0]; - const size = image.getSize(scaleFactor); - const buffer = image.toBitmap({ scaleFactor }); - representations.push({ scaleFactor, size, buffer }); - } else { - // Construct from dataURLs to ensure that they are not lost in creation. - for (const scaleFactor of scaleFactors) { - const size = image.getSize(scaleFactor); - const dataURL = image.toDataURL({ scaleFactor }); - representations.push({ scaleFactor, size, dataURL }); - } - } - return { __ELECTRON_SERIALIZED_NativeImage__: true, representations }; -} - -function deserializeNativeImage (value: any, createNativeImage: typeof Electron.nativeImage['createEmpty']) { - const image = createNativeImage(); - - // Use Buffer when there's only one representation for better perf. - // This avoids compressing to/from PNG where it's not necessary to - // ensure uniqueness of dataURLs (since there's only one). - if (value.representations.length === 1) { - const { buffer, size, scaleFactor } = value.representations[0]; - const { width, height } = size; - image.addRepresentation({ buffer, scaleFactor, width, height }); - } else { - // Construct from dataURLs to ensure that they are not lost in creation. - for (const rep of value.representations) { - const { dataURL, size, scaleFactor } = rep; - const { width, height } = size; - image.addRepresentation({ dataURL, scaleFactor, width, height }); - } - } - - return image; -} - -export function serialize (value: any): any { - if (value && value.constructor && value.constructor.name === 'NativeImage') { - return serializeNativeImage(value); - } if (Array.isArray(value)) { - return value.map(serialize); - } else if (isSerializableObject(value)) { - return value; - } else if (value instanceof Object) { - return objectMap(value, serialize); - } else { - return value; - } -} - -export function deserialize (value: any, createNativeImage: typeof Electron.nativeImage['createEmpty'] = getCreateNativeImage()): any { - if (value && value.__ELECTRON_SERIALIZED_NativeImage__) { - return deserializeNativeImage(value, createNativeImage); - } else if (Array.isArray(value)) { - return value.map(value => deserialize(value, createNativeImage)); - } else if (isSerializableObject(value)) { - return value; - } else if (value instanceof Object) { - return objectMap(value, value => deserialize(value, createNativeImage)); - } else { - return value; - } -} diff --git a/lib/common/web-view-methods.ts b/lib/common/web-view-methods.ts index a2cfb328cc4..baae378ddc0 100644 --- a/lib/common/web-view-methods.ts +++ b/lib/common/web-view-methods.ts @@ -59,6 +59,7 @@ export const properties = new Set([ ]); export const asyncMethods = new Set([ + 'capturePage', 'loadURL', 'executeJavaScript', 'insertCSS', diff --git a/lib/renderer/api/desktop-capturer.ts b/lib/renderer/api/desktop-capturer.ts index 3f138cc18d0..15a02e792d8 100644 --- a/lib/renderer/api/desktop-capturer.ts +++ b/lib/renderer/api/desktop-capturer.ts @@ -1,5 +1,4 @@ import { ipcRendererInternal } from '@electron/internal/renderer/ipc-renderer-internal'; -import { deserialize } from '@electron/internal/common/type-utils'; import deprecate from '@electron/internal/common/api/deprecate'; import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages'; @@ -21,5 +20,5 @@ export async function getSources (options: Electron.SourcesOptions) { deprecate.log('The use of \'desktopCapturer.getSources\' in the renderer process is deprecated and will be removed. See https://www.electronjs.org/docs/breaking-changes#removed-desktopcapturergetsources-in-the-renderer for more details.'); warned = true; } - return deserialize(await ipcRendererInternal.invoke(IPC_MESSAGES.DESKTOP_CAPTURER_GET_SOURCES, options, getCurrentStack())); + return await ipcRendererInternal.invoke(IPC_MESSAGES.DESKTOP_CAPTURER_GET_SOURCES, options, getCurrentStack()); } diff --git a/lib/renderer/web-view/guest-view-internal.ts b/lib/renderer/web-view/guest-view-internal.ts index b5a99f8744e..bd8a5a774a4 100644 --- a/lib/renderer/web-view/guest-view-internal.ts +++ b/lib/renderer/web-view/guest-view-internal.ts @@ -43,10 +43,6 @@ export function detachGuest (guestInstanceId: number) { return ipcRendererUtils.invokeSync(IPC_MESSAGES.GUEST_VIEW_MANAGER_DETACH_GUEST, guestInstanceId); } -export function capturePage (guestInstanceId: number, args: any[]) { - return ipcRendererInternal.invoke(IPC_MESSAGES.GUEST_VIEW_MANAGER_CAPTURE_PAGE, guestInstanceId, args); -} - export function invoke (guestInstanceId: number, method: string, args: any[]) { return ipcRendererInternal.invoke(IPC_MESSAGES.GUEST_VIEW_MANAGER_CALL, guestInstanceId, method, args); } diff --git a/lib/renderer/web-view/web-view-impl.ts b/lib/renderer/web-view/web-view-impl.ts index 958f7f52a2d..295879b3df8 100644 --- a/lib/renderer/web-view/web-view-impl.ts +++ b/lib/renderer/web-view/web-view-impl.ts @@ -3,7 +3,6 @@ import { WEB_VIEW_CONSTANTS } from '@electron/internal/renderer/web-view/web-vie import { syncMethods, asyncMethods, properties } from '@electron/internal/common/web-view-methods'; import type { WebViewAttribute, PartitionAttribute } from '@electron/internal/renderer/web-view/web-view-attributes'; import { setupWebViewAttributes } from '@electron/internal/renderer/web-view/web-view-attributes'; -import { deserialize } from '@electron/internal/common/type-utils'; // ID generator. let nextId = 0; @@ -16,7 +15,6 @@ export interface WebViewImplHooks { readonly guestViewInternal: typeof guestViewInternalModule; readonly allowGuestViewElementDefinition: NodeJS.InternalWebFrame['allowGuestViewElementDefinition']; readonly setIsWebView: (iframe: HTMLIFrameElement) => void; - readonly createNativeImage?: typeof Electron.nativeImage['createEmpty']; } // Represents the internal state of the WebView node. @@ -235,10 +233,6 @@ export const setupMethods = (WebViewElement: typeof ElectronInternal.WebViewElem }; } - WebViewElement.prototype.capturePage = async function (...args) { - return deserialize(await hooks.guestViewInternal.capturePage(this.getWebContentsId(), args), hooks.createNativeImage); - }; - const createPropertyGetter = function (property: string) { return function (this: ElectronInternal.WebViewElement) { return hooks.guestViewInternal.propertyGet(this.getWebContentsId(), property); diff --git a/shell/common/api/electron_api_native_image.h b/shell/common/api/electron_api_native_image.h index 76b13842036..09a869b58ec 100644 --- a/shell/common/api/electron_api_native_image.h +++ b/shell/common/api/electron_api_native_image.h @@ -45,6 +45,12 @@ namespace api { class NativeImage : public gin::Wrappable { public: + NativeImage(v8::Isolate* isolate, const gfx::Image& image); +#if defined(OS_WIN) + NativeImage(v8::Isolate* isolate, const base::FilePath& hicon_path); +#endif + ~NativeImage() override; + static gin::Handle CreateEmpty(v8::Isolate* isolate); static gin::Handle Create(v8::Isolate* isolate, const gfx::Image& image); @@ -95,13 +101,6 @@ class NativeImage : public gin::Wrappable { const gfx::Image& image() const { return image_; } - protected: - NativeImage(v8::Isolate* isolate, const gfx::Image& image); -#if defined(OS_WIN) - NativeImage(v8::Isolate* isolate, const base::FilePath& hicon_path); -#endif - ~NativeImage() override; - private: v8::Local ToPNG(gin::Arguments* args); v8::Local ToJPEG(v8::Isolate* isolate, int quality); diff --git a/shell/common/v8_value_serializer.cc b/shell/common/v8_value_serializer.cc index fdb04b88c3c..1876240378e 100644 --- a/shell/common/v8_value_serializer.cc +++ b/shell/common/v8_value_serializer.cc @@ -8,14 +8,17 @@ #include #include "gin/converter.h" +#include "shell/common/api/electron_api_native_image.h" #include "shell/common/gin_helper/microtasks_scope.h" +#include "skia/public/mojom/bitmap.mojom.h" #include "third_party/blink/public/common/messaging/cloneable_message.h" +#include "ui/gfx/image/image_skia.h" #include "v8/include/v8.h" namespace electron { namespace { -const uint8_t kVersionTag = 0xFF; +enum SerializationTag { kNativeImageTag = 'i', kVersionTag = 0xFF }; } // namespace class V8Serializer : public v8::ValueSerializer::Delegate { @@ -62,12 +65,35 @@ class V8Serializer : public v8::ValueSerializer::Delegate { data_ = {}; } + v8::Maybe WriteHostObject(v8::Isolate* isolate, + v8::Local object) override { + api::NativeImage* native_image; + if (gin::ConvertFromV8(isolate, object, &native_image)) { + // Serialize the NativeImage + WriteTag(kNativeImageTag); + gfx::ImageSkia image = native_image->image().AsImageSkia(); + std::vector image_reps = image.image_reps(); + serializer_.WriteUint32(image_reps.size()); + for (const auto& rep : image_reps) { + serializer_.WriteDouble(rep.scale()); + const SkBitmap& bitmap = rep.GetBitmap(); + std::vector bytes = + skia::mojom::InlineBitmap::Serialize(&bitmap); + serializer_.WriteUint32(bytes.size()); + serializer_.WriteRawBytes(bytes.data(), bytes.size()); + } + return v8::Just(true); + } else { + return v8::ValueSerializer::Delegate::WriteHostObject(isolate, object); + } + } + void ThrowDataCloneError(v8::Local message) override { isolate_->ThrowException(v8::Exception::Error(message)); } private: - void WriteTag(uint8_t tag) { serializer_.WriteRawBytes(&tag, 1); } + void WriteTag(SerializationTag tag) { serializer_.WriteRawBytes(&tag, 1); } void WriteBlinkEnvelope(uint32_t blink_version) { // Write a dummy blink version envelope for compatibility with @@ -107,6 +133,20 @@ class V8Deserializer : public v8::ValueDeserializer::Delegate { return scope.Escape(value); } + v8::MaybeLocal ReadHostObject(v8::Isolate* isolate) override { + uint8_t tag = 0; + if (!ReadTag(&tag)) + return v8::ValueDeserializer::Delegate::ReadHostObject(isolate); + switch (tag) { + case kNativeImageTag: + if (api::NativeImage* native_image = ReadNativeImage(isolate)) + return native_image->GetWrapper(isolate); + break; + } + // Throws an exception. + return v8::ValueDeserializer::Delegate::ReadHostObject(isolate); + } + private: bool ReadTag(uint8_t* tag) { const void* tag_bytes = nullptr; @@ -127,6 +167,31 @@ class V8Deserializer : public v8::ValueDeserializer::Delegate { return true; } + api::NativeImage* ReadNativeImage(v8::Isolate* isolate) { + gfx::ImageSkia image_skia; + uint32_t num_reps = 0; + if (!deserializer_.ReadUint32(&num_reps)) + return nullptr; + for (uint32_t i = 0; i < num_reps; i++) { + double scale = 0.0; + if (!deserializer_.ReadDouble(&scale)) + return nullptr; + uint32_t bitmap_size_bytes = 0; + if (!deserializer_.ReadUint32(&bitmap_size_bytes)) + return nullptr; + const void* bitmap_data = nullptr; + if (!deserializer_.ReadRawBytes(bitmap_size_bytes, &bitmap_data)) + return nullptr; + SkBitmap bitmap; + if (!skia::mojom::InlineBitmap::Deserialize(bitmap_data, + bitmap_size_bytes, &bitmap)) + return nullptr; + image_skia.AddRepresentation(gfx::ImageSkiaRep(bitmap, scale)); + } + gfx::Image image(image_skia); + return new api::NativeImage(isolate, image); + } + v8::Isolate* isolate_; v8::ValueDeserializer deserializer_; }; diff --git a/shell/renderer/renderer_client_base.cc b/shell/renderer/renderer_client_base.cc index 44084a82605..342e1daa3af 100644 --- a/shell/renderer/renderer_client_base.cc +++ b/shell/renderer/renderer_client_base.cc @@ -524,7 +524,6 @@ void RendererClientBase::SetupMainWorldOverrides( isolated_api.SetMethod("allowGuestViewElementDefinition", &AllowGuestViewElementDefinition); isolated_api.SetMethod("setIsWebView", &SetIsWebView); - isolated_api.SetMethod("createNativeImage", &api::NativeImage::CreateEmpty); auto source_context = GetContext(render_frame->GetWebFrame(), isolate); gin_helper::Dictionary global(isolate, source_context->Global()); diff --git a/typings/internal-ambient.d.ts b/typings/internal-ambient.d.ts index b0f91fa52fd..4f695936653 100644 --- a/typings/internal-ambient.d.ts +++ b/typings/internal-ambient.d.ts @@ -6,7 +6,6 @@ declare var isolatedApi: { guestViewInternal: any; allowGuestViewElementDefinition: NodeJS.InternalWebFrame['allowGuestViewElementDefinition']; setIsWebView: (iframe: HTMLIFrameElement) => void; - createNativeImage: typeof Electron.nativeImage['createEmpty']; } declare const BUILDFLAG: (flag: boolean) => boolean;