build: add v8_context_generator to mksnapshot zip (3-0-x) (#15505)

* build: add v8_context_generator to mksnapshot

Starting with 3-0-x, in order to use custom snapshots the v8_context_snapshot_generator binary is also needed.
Also, add tests for mksnapshot.

* Actually run verify-mksnapshot

.circleci/config.yml

@@ -70,6 +70,17 @@ build-steps: &build-steps
           else
             echo 'Headless testing not needed'
           fi
+
+    - run:
+        name: Verify mksnapshot
+        command: |
+          if [ "$RUN_TESTS" == "true" ] && [ "$ELECTRON_RELEASE" == "1" ]; then
+            echo 'Verifying mksnapshot on release build'
+            script/verify-mksnapshot.py
+          else
+            echo 'Skipping mksnapshot tests due to configuration'
+          fi
+
     - run:
         name: Test
         environment:

script/create-dist.py

@@ -79,6 +79,29 @@ TARGET_DIRECTORIES = {
   ],
 }
 
+V8_SNAPSHOT_BINARIES = {
+  'darwin': [
+    'libffmpeg.dylib',
+    'libicui18n.dylib',
+    'libicuuc.dylib',
+    'libv8.dylib',
+    'v8_context_snapshot_generator',
+  ],
+  'linux': [
+    'libffmpeg.so',
+    'libicui18n.so',
+    'libicuuc.so',
+    'libv8.so',
+    'v8_context_snapshot_generator',
+  ],
+  'win32': [
+    'ffmpeg.dll',
+    'icui18n.dll',
+    'icuuc.dll',
+    'v8.dll',
+    'v8_context_snapshot_generator.exe',
+  ],
+}
 
 def main():
   args = parse_args()
@@ -95,8 +118,8 @@ def main():
   force_build()
   create_symbols()
   copy_binaries()
-  copy_chrome_binary('chromedriver')
+  copy_chrome_binary('chromedriver', CHROMIUM_DIR, DIST_DIR)
-  copy_chrome_binary('mksnapshot')
+  copy_chrome_binary('mksnapshot', CHROMIUM_DIR, DIST_DIR)
   copy_license()
   if PLATFORM == 'win32':
     copy_vcruntime_binaries()
@@ -136,15 +159,21 @@ def copy_binaries():
                     symlinks=True)
 
 
-def copy_chrome_binary(binary):
+def copy_chrome_binary(binary, src_dir, dest_dir, is_native_mksnapshot = False):
   if PLATFORM == 'win32':
     binary += '.exe'
-  src = os.path.join(CHROMIUM_DIR, binary)
+  src = os.path.join(src_dir, binary)
-  dest = os.path.join(DIST_DIR, binary)
+  dest = os.path.join(dest_dir, binary)
 
   # Copy file and keep the executable bit.
   shutil.copyfile(src, dest)
   os.chmod(dest, os.stat(dest).st_mode | stat.S_IEXEC)
+  if binary.startswith('mksnapshot') and not is_native_mksnapshot:
+    snapshot_gen_path =  os.path.join(src_dir, 'snapshot_gen', '*')
+    snapshot_gen_files = glob.glob(snapshot_gen_path)
+    snapshot_gen_files += [ os.path.join(src_dir, get_ffmpeg_name()) ]
+    for gen_file in snapshot_gen_files:
+      shutil.copy2(gen_file, dest_dir)
 
 def copy_vcruntime_binaries():
   arch = get_target_arch()
@@ -272,9 +301,16 @@ def create_dist_zip():
 
 
 def create_chrome_binary_zip(binary, version):
+  files = ['LICENSE', 'LICENSES.chromium.html']
+  if PLATFORM == 'win32':
+    files += [binary + '.exe']
+  else:
+    files += [binary]
+
   file_suffix = ''
   create_native_mksnapshot = False
   if binary == 'mksnapshot':
+    files += V8_SNAPSHOT_BINARIES[PLATFORM]
     arch = get_target_arch()
     if arch.startswith('arm'):
       # if the arch is arm/arm64 the mksnapshot executable is an x64 binary,
@@ -284,23 +320,14 @@ def create_chrome_binary_zip(binary, version):
   dist_name = get_zip_name(binary, version, file_suffix)
   zip_file = os.path.join(SOURCE_ROOT, 'dist', dist_name)
 
-  files = ['LICENSE', 'LICENSES.chromium.html']
-  if PLATFORM == 'win32':
-    files += [binary + '.exe']
-  else:
-    files += [binary]
-
   with scoped_cwd(DIST_DIR):
     make_zip(zip_file, files, [])
 
   if create_native_mksnapshot == True:
     # Create a zip with the native version of the mksnapshot binary.
-    src = os.path.join(NATIVE_MKSNAPSHOT_DIR, binary)
+    copy_chrome_binary('mksnapshot', NATIVE_MKSNAPSHOT_DIR, DIST_DIR, True)
-    dest = os.path.join(DIST_DIR, binary)
+    copy_chrome_binary('v8_context_snapshot_generator', NATIVE_MKSNAPSHOT_DIR, \
-    # Copy file and keep the executable bit.
+                       DIST_DIR)
-    shutil.copyfile(src, dest)
-    os.chmod(dest, os.stat(dest).st_mode | stat.S_IEXEC)
-
     dist_name = get_zip_name(binary, version)
     zip_file = os.path.join(SOURCE_ROOT, 'dist', dist_name)
     with scoped_cwd(DIST_DIR):
@@ -310,13 +337,7 @@ def create_ffmpeg_zip():
   dist_name = get_zip_name('ffmpeg', ELECTRON_VERSION)
   zip_file = os.path.join(SOURCE_ROOT, 'dist', dist_name)
 
-  if PLATFORM == 'darwin':
+  ffmpeg_name = get_ffmpeg_name()
-    ffmpeg_name = 'libffmpeg.dylib'
-  elif PLATFORM == 'linux':
-    ffmpeg_name = 'libffmpeg.so'
-  elif PLATFORM == 'win32':
-    ffmpeg_name = 'ffmpeg.dll'
-
   shutil.copy2(os.path.join(CHROMIUM_DIR, '..', 'ffmpeg', ffmpeg_name),
                DIST_DIR)
 
@@ -326,6 +347,14 @@ def create_ffmpeg_zip():
   with scoped_cwd(DIST_DIR):
     make_zip(zip_file, [ffmpeg_name, 'LICENSE', 'LICENSES.chromium.html'], [])
 
+def get_ffmpeg_name():
+  if PLATFORM == 'darwin':
+    ffmpeg_name = 'libffmpeg.dylib'
+  elif PLATFORM == 'linux':
+    ffmpeg_name = 'libffmpeg.so'
+  elif PLATFORM == 'win32':
+    ffmpeg_name = 'ffmpeg.dll'
+  return ffmpeg_name
 
 def create_symbols_zip():
   if get_target_arch() == 'mips64el':

script/verify-mksnapshot.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+import glob
+import os
+import shutil
+import subprocess
+import sys
+
+from lib.util import electron_gyp, rm_rf, scoped_cwd
+
+
+SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+SNAPSHOT_DIR = os.path.join(SOURCE_ROOT, 'vendor', 'download',
+                                 'libchromiumcontent', 'static_library')
+PROJECT_NAME = electron_gyp()['project_name%']
+PRODUCT_NAME = electron_gyp()['product_name%']
+SNAPSHOT_SOURCE = os.path.join(SOURCE_ROOT, 'spec', 'fixtures', 'testsnap.js')
+
+def main():
+  os.chdir(SOURCE_ROOT)
+  app_path = create_app_copy()
+  blob_out_path = app_path
+  snapshot_gen_path =  os.path.join(SNAPSHOT_DIR, 'snapshot_gen', '*')
+  snapshot_gen_files = glob.glob(snapshot_gen_path)
+  if sys.platform == 'darwin':
+    electron = os.path.join(app_path, 'Contents', 'MacOS', PRODUCT_NAME)
+    blob_out_path = os.path.join(app_path, 'Contents', 'Frameworks',
+                    '{0} Framework.framework'.format(PROJECT_NAME),
+                    'Resources')
+    snapshot_gen_files += [ os.path.join(SNAPSHOT_DIR, 'libffmpeg.dylib') ]
+  elif sys.platform == 'win32':
+    electron = os.path.join(app_path, '{0}.exe'.format(PROJECT_NAME))
+    snapshot_gen_files += [
+      os.path.join(SNAPSHOT_DIR, 'ffmpeg.dll'),
+      os.path.join(SNAPSHOT_DIR, 'ffmpeg.dll.lib'),
+    ]
+  else:
+    electron = os.path.join(app_path, PROJECT_NAME)
+    snapshot_gen_files += [ os.path.join(SNAPSHOT_DIR, 'libffmpeg.so') ]
+
+  # Copy mksnapshot and friends to the directory the snapshot_blob should be
+  # generated in.
+  mksnapshot_binary = get_binary_path('mksnapshot', SNAPSHOT_DIR)
+  shutil.copy2(mksnapshot_binary, blob_out_path)
+  for gen_file in snapshot_gen_files:
+    shutil.copy2(gen_file, blob_out_path)
+
+  returncode = 0
+  try:
+    with scoped_cwd(blob_out_path):
+      mkargs = [ get_binary_path('mksnapshot', blob_out_path), \
+                 SNAPSHOT_SOURCE, '--startup_blob', 'snapshot_blob.bin' ]
+      subprocess.check_call(mkargs)
+      print 'ok mksnapshot successfully created snapshot_blob.bin.'
+      context_snapshot = 'v8_context_snapshot.bin'
+      context_snapshot_path = os.path.join(blob_out_path, context_snapshot)
+      gen_binary = get_binary_path('v8_context_snapshot_generator', \
+                                   blob_out_path)
+      genargs = [ gen_binary, \
+                 '--output_file={0}'.format(context_snapshot_path) ]
+      subprocess.check_call(genargs)
+      print 'ok v8_context_snapshot_generator successfully created ' \
+            + context_snapshot
+
+    test_path = os.path.join('spec', 'fixtures', 'snapshot-items-available.js')
+    subprocess.check_call([electron, test_path])
+    print 'ok successfully used custom snapshot.'
+  except subprocess.CalledProcessError as e:
+    print 'not ok an error was encountered while testing mksnapshot.'
+    print e
+    returncode = e.returncode
+  except KeyboardInterrupt:
+    print 'Other error'
+    returncode = 0
+
+  return returncode
+
+
+# Create copy of app to create custom snapshot
+def create_app_copy():
+  initial_app_path = os.path.join(SOURCE_ROOT, 'out', 'R')
+  app_path = os.path.join(SOURCE_ROOT, 'out', 'R-mksnapshot-test')
+
+  if sys.platform == 'darwin':
+    app_name = '{0}.app'.format(PRODUCT_NAME)
+    initial_app_path = os.path.join(initial_app_path, app_name)
+    app_path = os.path.join(app_path, app_name)
+
+  rm_rf(app_path)
+  shutil.copytree(initial_app_path, app_path, symlinks=True)
+  return app_path
+
+def get_binary_path(binary_name, root_path):
+  if sys.platform == 'win32':
+    binary_path = os.path.join(root_path, '{0}.exe'.format(binary_name))
+  else:
+    binary_path = os.path.join(root_path, binary_name)
+  return binary_path
+
+if __name__ == '__main__':
+  sys.exit(main())

spec/fixtures/snapshot-items-available.js

@@ -0,0 +1,20 @@
+// Verifies that objects contained in custom snapshot are accessible in Electron.
+
+const {app} = require('electron')
+
+app.once('ready', () => {
+  try {
+    const testValue = f() // eslint-disable-line no-undef
+    if (testValue === 86) {
+      console.log('ok test snapshot successfully loaded.')
+      app.exit(0)
+    } else {
+      console.log('not ok test snapshot could not be successfully loaded.')
+      app.exit(1)
+    }
+    return
+  } catch (ex) {
+    console.log('Error running custom snapshot', ex)
+    app.exit(1)
+  }
+})

spec/fixtures/testsnap.js

@@ -0,0 +1,3 @@
+// taken from https://chromium.googlesource.com/v8/v8.git/+/HEAD/test/cctest/test-serialize.cc#1127
+function f () { return g() * 2 } // eslint-disable-line no-unused-vars
+function g () { return 43 }