feat: serialize NativeImage over ipc (#30729)

This commit is contained in:
Jeremy Rose 2021-09-07 10:37:45 -07:00 committed by GitHub
parent ee33374675
commit 55c57808fb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 79 additions and 157 deletions

View file

@ -138,7 +138,6 @@ auto_filenames = {
"lib/common/api/native-image.ts",
"lib/common/define-properties.ts",
"lib/common/ipc-messages.ts",
"lib/common/type-utils.ts",
"lib/common/web-view-methods.ts",
"lib/renderer/api/context-bridge.ts",
"lib/renderer/api/crash-reporter.ts",
@ -168,7 +167,6 @@ auto_filenames = {
]
isolated_bundle_deps = [
"lib/common/type-utils.ts",
"lib/common/web-view-methods.ts",
"lib/isolated_renderer/init.ts",
"lib/renderer/web-view/web-view-attributes.ts",
@ -246,7 +244,6 @@ auto_filenames = {
"lib/common/ipc-messages.ts",
"lib/common/parse-features-string.ts",
"lib/common/reset-search-paths.ts",
"lib/common/type-utils.ts",
"lib/common/web-view-events.ts",
"lib/common/web-view-methods.ts",
"lib/common/webpack-globals-provider.ts",
@ -269,7 +266,6 @@ auto_filenames = {
"lib/common/init.ts",
"lib/common/ipc-messages.ts",
"lib/common/reset-search-paths.ts",
"lib/common/type-utils.ts",
"lib/common/web-view-methods.ts",
"lib/common/webpack-provider.ts",
"lib/renderer/api/context-bridge.ts",
@ -309,7 +305,6 @@ auto_filenames = {
"lib/common/init.ts",
"lib/common/ipc-messages.ts",
"lib/common/reset-search-paths.ts",
"lib/common/type-utils.ts",
"lib/common/webpack-provider.ts",
"lib/renderer/api/context-bridge.ts",
"lib/renderer/api/crash-reporter.ts",

View file

@ -4,7 +4,6 @@ import * as ipcMainUtils from '@electron/internal/browser/ipc-main-internal-util
import { parseWebViewWebPreferences } from '@electron/internal/common/parse-features-string';
import { syncMethods, asyncMethods, properties } from '@electron/internal/common/web-view-methods';
import { webViewEvents } from '@electron/internal/common/web-view-events';
import { serialize } from '@electron/internal/common/type-utils';
import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages';
interface GuestInstance {
@ -330,12 +329,6 @@ handleMessageSync(IPC_MESSAGES.GUEST_VIEW_MANAGER_PROPERTY_SET, function (event,
(guest as any)[property] = val;
});
handleMessage(IPC_MESSAGES.GUEST_VIEW_MANAGER_CAPTURE_PAGE, async function (event, guestInstanceId: number, args: any[]) {
const guest = getGuestForWebContents(guestInstanceId, event.sender);
return serialize(await guest.capturePage(...args));
});
// Returns WebContents from its guest id hosted in given webContents.
const getGuestForWebContents = function (guestInstanceId: number, contents: Electron.WebContents) {
const guestInstance = guestInstances.get(guestInstanceId);

View file

@ -4,7 +4,6 @@ import { clipboard } from 'electron/common';
import * as fs from 'fs';
import { ipcMainInternal } from '@electron/internal/browser/ipc-main-internal';
import * as ipcMainUtils from '@electron/internal/browser/ipc-main-internal-utils';
import * as typeUtils from '@electron/internal/common/type-utils';
import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages';
import type * as desktopCapturerModule from '@electron/internal/browser/desktop-capturer';
@ -56,7 +55,7 @@ ipcMainUtils.handleSync(IPC_MESSAGES.BROWSER_CLIPBOARD_SYNC, function (event, me
throw new Error(`Invalid method: ${method}`);
}
return typeUtils.serialize((clipboard as any)[method](...typeUtils.deserialize(args)));
return (clipboard as any)[method](...args);
});
if (BUILDFLAG(ENABLE_DESKTOP_CAPTURER)) {
@ -71,7 +70,7 @@ if (BUILDFLAG(ENABLE_DESKTOP_CAPTURER)) {
return [];
}
return typeUtils.serialize(await desktopCapturer.getSourcesImpl(event.sender, options));
return await desktopCapturer.getSourcesImpl(event.sender, options);
});
}

View file

@ -1,20 +1,14 @@
import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages';
import type * as ipcRendererUtilsModule from '@electron/internal/renderer/ipc-renderer-internal-utils';
import type * as typeUtilsModule from '@electron/internal/common/type-utils';
const clipboard = process._linkedBinding('electron_common_clipboard');
if (process.type === 'renderer') {
const ipcRendererUtils = require('@electron/internal/renderer/ipc-renderer-internal-utils') as typeof ipcRendererUtilsModule;
const typeUtils = require('@electron/internal/common/type-utils') as typeof typeUtilsModule;
const makeRemoteMethod = function (method: keyof Electron.Clipboard) {
return (...args: any[]) => {
args = typeUtils.serialize(args);
const result = ipcRendererUtils.invokeSync(IPC_MESSAGES.BROWSER_CLIPBOARD_SYNC, method, ...args);
return typeUtils.deserialize(result);
};
const makeRemoteMethod = function (method: keyof Electron.Clipboard): any {
return (...args: any[]) => ipcRendererUtils.invokeSync(IPC_MESSAGES.BROWSER_CLIPBOARD_SYNC, method, ...args);
};
if (process.platform === 'linux') {

View file

@ -13,7 +13,6 @@ export const enum IPC_MESSAGES {
GUEST_VIEW_MANAGER_DETACH_GUEST = 'GUEST_VIEW_MANAGER_DETACH_GUEST',
GUEST_VIEW_MANAGER_FOCUS_CHANGE = 'GUEST_VIEW_MANAGER_FOCUS_CHANGE',
GUEST_VIEW_MANAGER_CALL = 'GUEST_VIEW_MANAGER_CALL',
GUEST_VIEW_MANAGER_CAPTURE_PAGE = 'GUEST_VIEW_MANAGER_CAPTURE_PAGE',
GUEST_VIEW_MANAGER_PROPERTY_GET = 'GUEST_VIEW_MANAGER_PROPERTY_GET',
GUEST_VIEW_MANAGER_PROPERTY_SET = 'GUEST_VIEW_MANAGER_PROPERTY_SET',

View file

@ -1,110 +0,0 @@
function getCreateNativeImage () {
return process._linkedBinding('electron_common_native_image').nativeImage.createEmpty;
}
export function isPromise (val: any) {
return (
val &&
val.then &&
val.then instanceof Function &&
val.constructor &&
val.constructor.reject &&
val.constructor.reject instanceof Function &&
val.constructor.resolve &&
val.constructor.resolve instanceof Function
);
}
const serializableTypes = [
Boolean,
Number,
String,
Date,
Error,
RegExp,
ArrayBuffer
];
// https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm#Supported_types
export function isSerializableObject (value: any) {
return value === null || ArrayBuffer.isView(value) || serializableTypes.some(type => value instanceof type);
}
const objectMap = function (source: Object, mapper: (value: any) => any) {
const sourceEntries = Object.entries(source);
const targetEntries = sourceEntries.map(([key, val]) => [key, mapper(val)]);
return Object.fromEntries(targetEntries);
};
function serializeNativeImage (image: Electron.NativeImage) {
const representations = [];
const scaleFactors = image.getScaleFactors();
// Use Buffer when there's only one representation for better perf.
// This avoids compressing to/from PNG where it's not necessary to
// ensure uniqueness of dataURLs (since there's only one).
if (scaleFactors.length === 1) {
const scaleFactor = scaleFactors[0];
const size = image.getSize(scaleFactor);
const buffer = image.toBitmap({ scaleFactor });
representations.push({ scaleFactor, size, buffer });
} else {
// Construct from dataURLs to ensure that they are not lost in creation.
for (const scaleFactor of scaleFactors) {
const size = image.getSize(scaleFactor);
const dataURL = image.toDataURL({ scaleFactor });
representations.push({ scaleFactor, size, dataURL });
}
}
return { __ELECTRON_SERIALIZED_NativeImage__: true, representations };
}
function deserializeNativeImage (value: any, createNativeImage: typeof Electron.nativeImage['createEmpty']) {
const image = createNativeImage();
// Use Buffer when there's only one representation for better perf.
// This avoids compressing to/from PNG where it's not necessary to
// ensure uniqueness of dataURLs (since there's only one).
if (value.representations.length === 1) {
const { buffer, size, scaleFactor } = value.representations[0];
const { width, height } = size;
image.addRepresentation({ buffer, scaleFactor, width, height });
} else {
// Construct from dataURLs to ensure that they are not lost in creation.
for (const rep of value.representations) {
const { dataURL, size, scaleFactor } = rep;
const { width, height } = size;
image.addRepresentation({ dataURL, scaleFactor, width, height });
}
}
return image;
}
export function serialize (value: any): any {
if (value && value.constructor && value.constructor.name === 'NativeImage') {
return serializeNativeImage(value);
} if (Array.isArray(value)) {
return value.map(serialize);
} else if (isSerializableObject(value)) {
return value;
} else if (value instanceof Object) {
return objectMap(value, serialize);
} else {
return value;
}
}
export function deserialize (value: any, createNativeImage: typeof Electron.nativeImage['createEmpty'] = getCreateNativeImage()): any {
if (value && value.__ELECTRON_SERIALIZED_NativeImage__) {
return deserializeNativeImage(value, createNativeImage);
} else if (Array.isArray(value)) {
return value.map(value => deserialize(value, createNativeImage));
} else if (isSerializableObject(value)) {
return value;
} else if (value instanceof Object) {
return objectMap(value, value => deserialize(value, createNativeImage));
} else {
return value;
}
}

View file

@ -59,6 +59,7 @@ export const properties = new Set([
]);
export const asyncMethods = new Set([
'capturePage',
'loadURL',
'executeJavaScript',
'insertCSS',

View file

@ -1,5 +1,4 @@
import { ipcRendererInternal } from '@electron/internal/renderer/ipc-renderer-internal';
import { deserialize } from '@electron/internal/common/type-utils';
import deprecate from '@electron/internal/common/api/deprecate';
import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages';
@ -21,5 +20,5 @@ export async function getSources (options: Electron.SourcesOptions) {
deprecate.log('The use of \'desktopCapturer.getSources\' in the renderer process is deprecated and will be removed. See https://www.electronjs.org/docs/breaking-changes#removed-desktopcapturergetsources-in-the-renderer for more details.');
warned = true;
}
return deserialize(await ipcRendererInternal.invoke(IPC_MESSAGES.DESKTOP_CAPTURER_GET_SOURCES, options, getCurrentStack()));
return await ipcRendererInternal.invoke(IPC_MESSAGES.DESKTOP_CAPTURER_GET_SOURCES, options, getCurrentStack());
}

View file

@ -43,10 +43,6 @@ export function detachGuest (guestInstanceId: number) {
return ipcRendererUtils.invokeSync(IPC_MESSAGES.GUEST_VIEW_MANAGER_DETACH_GUEST, guestInstanceId);
}
export function capturePage (guestInstanceId: number, args: any[]) {
return ipcRendererInternal.invoke(IPC_MESSAGES.GUEST_VIEW_MANAGER_CAPTURE_PAGE, guestInstanceId, args);
}
export function invoke (guestInstanceId: number, method: string, args: any[]) {
return ipcRendererInternal.invoke(IPC_MESSAGES.GUEST_VIEW_MANAGER_CALL, guestInstanceId, method, args);
}

View file

@ -3,7 +3,6 @@ import { WEB_VIEW_CONSTANTS } from '@electron/internal/renderer/web-view/web-vie
import { syncMethods, asyncMethods, properties } from '@electron/internal/common/web-view-methods';
import type { WebViewAttribute, PartitionAttribute } from '@electron/internal/renderer/web-view/web-view-attributes';
import { setupWebViewAttributes } from '@electron/internal/renderer/web-view/web-view-attributes';
import { deserialize } from '@electron/internal/common/type-utils';
// ID generator.
let nextId = 0;
@ -16,7 +15,6 @@ export interface WebViewImplHooks {
readonly guestViewInternal: typeof guestViewInternalModule;
readonly allowGuestViewElementDefinition: NodeJS.InternalWebFrame['allowGuestViewElementDefinition'];
readonly setIsWebView: (iframe: HTMLIFrameElement) => void;
readonly createNativeImage?: typeof Electron.nativeImage['createEmpty'];
}
// Represents the internal state of the WebView node.
@ -235,10 +233,6 @@ export const setupMethods = (WebViewElement: typeof ElectronInternal.WebViewElem
};
}
WebViewElement.prototype.capturePage = async function (...args) {
return deserialize(await hooks.guestViewInternal.capturePage(this.getWebContentsId(), args), hooks.createNativeImage);
};
const createPropertyGetter = function (property: string) {
return function (this: ElectronInternal.WebViewElement) {
return hooks.guestViewInternal.propertyGet(this.getWebContentsId(), property);

View file

@ -45,6 +45,12 @@ namespace api {
class NativeImage : public gin::Wrappable<NativeImage> {
public:
NativeImage(v8::Isolate* isolate, const gfx::Image& image);
#if defined(OS_WIN)
NativeImage(v8::Isolate* isolate, const base::FilePath& hicon_path);
#endif
~NativeImage() override;
static gin::Handle<NativeImage> CreateEmpty(v8::Isolate* isolate);
static gin::Handle<NativeImage> Create(v8::Isolate* isolate,
const gfx::Image& image);
@ -95,13 +101,6 @@ class NativeImage : public gin::Wrappable<NativeImage> {
const gfx::Image& image() const { return image_; }
protected:
NativeImage(v8::Isolate* isolate, const gfx::Image& image);
#if defined(OS_WIN)
NativeImage(v8::Isolate* isolate, const base::FilePath& hicon_path);
#endif
~NativeImage() override;
private:
v8::Local<v8::Value> ToPNG(gin::Arguments* args);
v8::Local<v8::Value> ToJPEG(v8::Isolate* isolate, int quality);

View file

@ -8,14 +8,17 @@
#include <vector>
#include "gin/converter.h"
#include "shell/common/api/electron_api_native_image.h"
#include "shell/common/gin_helper/microtasks_scope.h"
#include "skia/public/mojom/bitmap.mojom.h"
#include "third_party/blink/public/common/messaging/cloneable_message.h"
#include "ui/gfx/image/image_skia.h"
#include "v8/include/v8.h"
namespace electron {
namespace {
const uint8_t kVersionTag = 0xFF;
enum SerializationTag { kNativeImageTag = 'i', kVersionTag = 0xFF };
} // namespace
class V8Serializer : public v8::ValueSerializer::Delegate {
@ -62,12 +65,35 @@ class V8Serializer : public v8::ValueSerializer::Delegate {
data_ = {};
}
v8::Maybe<bool> WriteHostObject(v8::Isolate* isolate,
v8::Local<v8::Object> object) override {
api::NativeImage* native_image;
if (gin::ConvertFromV8(isolate, object, &native_image)) {
// Serialize the NativeImage
WriteTag(kNativeImageTag);
gfx::ImageSkia image = native_image->image().AsImageSkia();
std::vector<gfx::ImageSkiaRep> image_reps = image.image_reps();
serializer_.WriteUint32(image_reps.size());
for (const auto& rep : image_reps) {
serializer_.WriteDouble(rep.scale());
const SkBitmap& bitmap = rep.GetBitmap();
std::vector<uint8_t> bytes =
skia::mojom::InlineBitmap::Serialize(&bitmap);
serializer_.WriteUint32(bytes.size());
serializer_.WriteRawBytes(bytes.data(), bytes.size());
}
return v8::Just(true);
} else {
return v8::ValueSerializer::Delegate::WriteHostObject(isolate, object);
}
}
void ThrowDataCloneError(v8::Local<v8::String> message) override {
isolate_->ThrowException(v8::Exception::Error(message));
}
private:
void WriteTag(uint8_t tag) { serializer_.WriteRawBytes(&tag, 1); }
void WriteTag(SerializationTag tag) { serializer_.WriteRawBytes(&tag, 1); }
void WriteBlinkEnvelope(uint32_t blink_version) {
// Write a dummy blink version envelope for compatibility with
@ -107,6 +133,20 @@ class V8Deserializer : public v8::ValueDeserializer::Delegate {
return scope.Escape(value);
}
v8::MaybeLocal<v8::Object> ReadHostObject(v8::Isolate* isolate) override {
uint8_t tag = 0;
if (!ReadTag(&tag))
return v8::ValueDeserializer::Delegate::ReadHostObject(isolate);
switch (tag) {
case kNativeImageTag:
if (api::NativeImage* native_image = ReadNativeImage(isolate))
return native_image->GetWrapper(isolate);
break;
}
// Throws an exception.
return v8::ValueDeserializer::Delegate::ReadHostObject(isolate);
}
private:
bool ReadTag(uint8_t* tag) {
const void* tag_bytes = nullptr;
@ -127,6 +167,31 @@ class V8Deserializer : public v8::ValueDeserializer::Delegate {
return true;
}
api::NativeImage* ReadNativeImage(v8::Isolate* isolate) {
gfx::ImageSkia image_skia;
uint32_t num_reps = 0;
if (!deserializer_.ReadUint32(&num_reps))
return nullptr;
for (uint32_t i = 0; i < num_reps; i++) {
double scale = 0.0;
if (!deserializer_.ReadDouble(&scale))
return nullptr;
uint32_t bitmap_size_bytes = 0;
if (!deserializer_.ReadUint32(&bitmap_size_bytes))
return nullptr;
const void* bitmap_data = nullptr;
if (!deserializer_.ReadRawBytes(bitmap_size_bytes, &bitmap_data))
return nullptr;
SkBitmap bitmap;
if (!skia::mojom::InlineBitmap::Deserialize(bitmap_data,
bitmap_size_bytes, &bitmap))
return nullptr;
image_skia.AddRepresentation(gfx::ImageSkiaRep(bitmap, scale));
}
gfx::Image image(image_skia);
return new api::NativeImage(isolate, image);
}
v8::Isolate* isolate_;
v8::ValueDeserializer deserializer_;
};

View file

@ -524,7 +524,6 @@ void RendererClientBase::SetupMainWorldOverrides(
isolated_api.SetMethod("allowGuestViewElementDefinition",
&AllowGuestViewElementDefinition);
isolated_api.SetMethod("setIsWebView", &SetIsWebView);
isolated_api.SetMethod("createNativeImage", &api::NativeImage::CreateEmpty);
auto source_context = GetContext(render_frame->GetWebFrame(), isolate);
gin_helper::Dictionary global(isolate, source_context->Global());

View file

@ -6,7 +6,6 @@ declare var isolatedApi: {
guestViewInternal: any;
allowGuestViewElementDefinition: NodeJS.InternalWebFrame['allowGuestViewElementDefinition'];
setIsWebView: (iframe: HTMLIFrameElement) => void;
createNativeImage: typeof Electron.nativeImage['createEmpty'];
}
declare const BUILDFLAG: (flag: boolean) => boolean;