refactor: correctly serialize nativeImage/buffer with typeUtils (#23666)

* refactor: correctly serialize nativeImage/buffer with typeUtils

* test: add serialization specs

* fix: construct from dataURL

* test: test for dataURL specificity

lib/browser/remote/server.ts

@@ -245,7 +245,7 @@ type MetaTypeFromRenderer = {
   length: number
 } | {
   type: 'nativeimage',
-  value: { size: Size, buffer: Buffer, scaleFactor: number }[]
+  value: { size: Size, buffer: Buffer, scaleFactor: number, dataURL: string }[]
 }
 
 const fakeConstructor = (constructor: Function, name: string) =>
@@ -266,13 +266,9 @@ const unwrapArgs = function (sender: electron.WebContents, frameId: number, cont
       case 'nativeimage': {
         const image = electron.nativeImage.createEmpty();
         for (const rep of meta.value) {
-          const { buffer, size, scaleFactor } = rep;
+          const { size, scaleFactor, dataURL } = rep;
-          image.addRepresentation({
+          const { width, height } = size;
-            buffer,
+          image.addRepresentation({ dataURL, scaleFactor, width, height });
-            width: size.width,
-            height: size.height,
-            scaleFactor
-          });
         }
         return image;
       }

lib/common/type-utils.ts

@@ -20,7 +20,8 @@ const serializableTypes = [
   Date,
   Error,
   RegExp,
-  ArrayBuffer
+  ArrayBuffer,
+  NativeImage
 ];
 
 export function isSerializableObject (value: any) {
@@ -35,11 +36,15 @@ const objectMap = function (source: Object, mapper: (value: any) => any) {
 
 export function serialize (value: any): any {
   if (value instanceof NativeImage) {
-    return {
+    const representations = [];
-      buffer: value.toBitmap(),
+    for (const scaleFactor of value.getScaleFactors()) {
-      size: value.getSize(),
+      const size = value.getSize(scaleFactor);
-      __ELECTRON_SERIALIZED_NativeImage__: true
+      const dataURL = value.toDataURL({ scaleFactor });
-    };
+      representations.push({ scaleFactor, size, dataURL });
+    }
+    return { __ELECTRON_SERIALIZED_NativeImage__: true, representations };
+  } else if (value instanceof Buffer) {
+    return { __ELECTRON_SERIALIZED_Buffer__: true, data: value };
   } else if (Array.isArray(value)) {
     return value.map(serialize);
   } else if (isSerializableObject(value)) {
@@ -53,7 +58,16 @@ export function serialize (value: any): any {
 
 export function deserialize (value: any): any {
   if (value && value.__ELECTRON_SERIALIZED_NativeImage__) {
-    return nativeImage.createFromBitmap(value.buffer, value.size);
+    const image = nativeImage.createEmpty();
+    for (const rep of value.representations) {
+      const { size, scaleFactor, dataURL } = rep;
+      const { width, height } = size;
+      image.addRepresentation({ dataURL, scaleFactor, width, height });
+    }
+    return image;
+  } else if (value && value.__ELECTRON_SERIALIZED_Buffer__) {
+    const { buffer, byteOffset, byteLength } = value.data;
+    return Buffer.from(buffer, byteOffset, byteLength);
   } else if (Array.isArray(value)) {
     return value.map(deserialize);
   } else if (isSerializableObject(value)) {

lib/renderer/api/remote.js

@@ -41,7 +41,8 @@ function wrapArgs (args, visited = new Set()) {
       for (const scaleFactor of value.getScaleFactors()) {
         const size = value.getSize(scaleFactor);
         const buffer = value.toBitmap({ scaleFactor });
-        images.push({ buffer, scaleFactor, size });
+        const dataURL = value.toDataURL({ scaleFactor });
+        images.push({ buffer, scaleFactor, size, dataURL });
       }
       return { type: 'nativeimage', value: images };
     } else if (Array.isArray(value)) {

spec-main/api-remote-spec.ts

@@ -6,6 +6,8 @@ import { ifdescribe } from './spec-helpers';
 import { ipcMain, BrowserWindow } from 'electron/main';
 import { emittedOnce } from './events-helpers';
 import { NativeImage } from 'electron/common';
+import { serialize, deserialize } from '../lib/common/type-utils';
+import { nativeImage } from 'electron';
 
 const features = process.electronBinding('features');
 
@@ -76,6 +78,143 @@ function makeEachWindow () {
   return () => w;
 }
 
+describe('typeUtils serialization/deserialization', () => {
+  it('serializes and deserializes an empty NativeImage', () => {
+    const image = nativeImage.createEmpty();
+    const serializedImage = serialize(image);
+    const empty = deserialize(serializedImage);
+
+    expect(empty.isEmpty()).to.be.true();
+    expect(empty.getAspectRatio()).to.equal(1);
+    expect(empty.toDataURL()).to.equal('data:image/png;base64,');
+    expect(empty.toDataURL({ scaleFactor: 2.0 })).to.equal('data:image/png;base64,');
+    expect(empty.getSize()).to.deep.equal({ width: 0, height: 0 });
+    expect(empty.getBitmap()).to.be.empty();
+    expect(empty.getBitmap({ scaleFactor: 2.0 })).to.be.empty();
+    expect(empty.toBitmap()).to.be.empty();
+    expect(empty.toBitmap({ scaleFactor: 2.0 })).to.be.empty();
+    expect(empty.toJPEG(100)).to.be.empty();
+    expect(empty.toPNG()).to.be.empty();
+    expect(empty.toPNG({ scaleFactor: 2.0 })).to.be.empty();
+  });
+
+  it('serializes and deserializes a non-empty NativeImage', () => {
+    const dataURL = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAFklEQVQYlWP8//8/AwMDEwMDAwMDAwAkBgMBBMzldwAAAABJRU5ErkJggg==';
+    const image = nativeImage.createFromDataURL(dataURL);
+    const serializedImage = serialize(image);
+    const nonEmpty = deserialize(serializedImage);
+
+    expect(nonEmpty.isEmpty()).to.be.false();
+    expect(nonEmpty.getAspectRatio()).to.equal(1);
+    expect(nonEmpty.toDataURL()).to.not.be.empty();
+    expect(nonEmpty.toDataURL({ scaleFactor: 1.0 })).to.equal(dataURL);
+    expect(nonEmpty.getSize()).to.deep.equal({ width: 2, height: 2 });
+    expect(nonEmpty.getBitmap()).to.not.be.empty();
+    expect(nonEmpty.toPNG()).to.not.be.empty();
+  });
+
+  it('serializes and deserializes a non-empty NativeImage with multiple representations', () => {
+    const image = nativeImage.createEmpty();
+
+    const dataURL1 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVQYlWNgAAIAAAUAAdafFs0AAAAASUVORK5CYII=';
+    image.addRepresentation({ scaleFactor: 1.0, dataURL: dataURL1 });
+
+    const dataURL2 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAFklEQVQYlWP8//8/AwMDEwMDAwMDAwAkBgMBBMzldwAAAABJRU5ErkJggg==';
+    image.addRepresentation({ scaleFactor: 2.0, dataURL: dataURL2 });
+
+    const serializedImage = serialize(image);
+    const nonEmpty = deserialize(serializedImage);
+
+    expect(nonEmpty.isEmpty()).to.be.false();
+    expect(nonEmpty.getAspectRatio()).to.equal(1);
+    expect(nonEmpty.getSize()).to.deep.equal({ width: 1, height: 1 });
+    expect(nonEmpty.getBitmap()).to.not.be.empty();
+    expect(nonEmpty.getBitmap({ scaleFactor: 1.0 })).to.not.be.empty();
+    expect(nonEmpty.getBitmap({ scaleFactor: 2.0 })).to.not.be.empty();
+    expect(nonEmpty.toBitmap()).to.not.be.empty();
+    expect(nonEmpty.toBitmap({ scaleFactor: 1.0 })).to.not.be.empty();
+    expect(nonEmpty.toBitmap({ scaleFactor: 2.0 })).to.not.be.empty();
+    expect(nonEmpty.toPNG()).to.not.be.empty();
+    expect(nonEmpty.toPNG({ scaleFactor: 1.0 })).to.not.be.empty();
+    expect(nonEmpty.toPNG({ scaleFactor: 2.0 })).to.not.be.empty();
+    expect(nonEmpty.toDataURL()).to.not.be.empty();
+    expect(nonEmpty.toDataURL({ scaleFactor: 1.0 })).to.equal(dataURL1);
+    expect(nonEmpty.toDataURL({ scaleFactor: 2.0 })).to.equal(dataURL2);
+  });
+
+  it('serializes and deserializes an Array', () => {
+    const array = [1, 2, 3, 4, 5];
+    const serialized = serialize(array);
+    const deserialized = deserialize(serialized);
+
+    expect(deserialized).to.deep.equal(array);
+  });
+
+  it('serializes and deserializes a Buffer', () => {
+    const buffer = Buffer.from('hello world!', 'utf-8');
+    const serialized = serialize(buffer);
+    const deserialized = deserialize(serialized);
+
+    expect(deserialized).to.deep.equal(buffer);
+  });
+
+  it('serializes and deserializes a Boolean', () => {
+    const bool = true;
+    const serialized = serialize(bool);
+    const deserialized = deserialize(serialized);
+
+    expect(deserialized).to.equal(bool);
+  });
+
+  it('serializes and deserializes a Date', () => {
+    const date = new Date();
+    const serialized = serialize(date);
+    const deserialized = deserialize(serialized);
+
+    expect(deserialized).to.equal(date);
+  });
+
+  it('serializes and deserializes a Number', () => {
+    const number = 42;
+    const serialized = serialize(number);
+    const deserialized = deserialize(serialized);
+
+    expect(deserialized).to.equal(number);
+  });
+
+  it('serializes and deserializes a Regexp', () => {
+    const regex = new RegExp('ab+c');
+    const serialized = serialize(regex);
+    const deserialized = deserialize(serialized);
+
+    expect(deserialized).to.equal(regex);
+  });
+
+  it('serializes and deserializes a String', () => {
+    const str = 'hello world';
+    const serialized = serialize(str);
+    const deserialized = deserialize(serialized);
+
+    expect(deserialized).to.equal(str);
+  });
+
+  it('serializes and deserializes an Error', () => {
+    const err = new Error('oh crap');
+    const serialized = serialize(err);
+    const deserialized = deserialize(serialized);
+
+    expect(deserialized).to.equal(err);
+  });
+
+  it('serializes and deserializes a simple Object', () => {
+    const obj = { hello: 'world', 'answer-to-everything': 42 };
+    const serialized = serialize(obj);
+    const deserialized = deserialize(serialized);
+
+    expect(deserialized).to.deep.equal(obj);
+  });
+});
+
 ifdescribe(features.isRemoteModuleEnabled())('remote module', () => {
   const fixtures = path.join(__dirname, 'fixtures');