Merge pull request #9589 from electron/rebuild-before-specs

Rebuild native modules before specs run

package.json

@@ -27,9 +27,10 @@
     "browserify": "browserify",
     "bump-version": "./script/bump-version.py",
     "build": "python ./script/build.py -c D",
+    "rebuild-test-modules": "python ./script/rebuild-test-modules.py",
     "clean": "python ./script/clean.py",
     "clean-build": "python ./script/clean.py --build",
-    "coverage": "npm run instrument-code-coverage && npm test -- --use-instrumented-asar",
+    "coverage": "npm run instrument-code-coverage && npm test -- --use_instrumented_asar",
     "instrument-code-coverage": "electabul instrument --input-path ./lib --output-path ./out/coverage/electron.asar",
     "lint": "npm run lint-js && npm run lint-cpp && npm run lint-py && npm run lint-api-docs-js && npm run lint-api-docs",
     "lint-js": "standard && cd spec && standard",

script/bootstrap.py

@@ -9,7 +9,8 @@ import sys
 
 from lib.config import BASE_URL, PLATFORM,  enable_verbose_mode, \
                        is_verbose_mode, get_target_arch
-from lib.util import execute, execute_stdout, get_electron_version, scoped_cwd
+from lib.util import execute, execute_stdout, get_electron_version, \
+                     scoped_cwd, update_node_modules
 
 
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
@@ -17,10 +18,6 @@ VENDOR_DIR = os.path.join(SOURCE_ROOT, 'vendor')
 DOWNLOAD_DIR = os.path.join(VENDOR_DIR, 'download')
 PYTHON_26_URL = 'https://chromium.googlesource.com/chromium/deps/python_26'
 
-NPM = 'npm'
-if sys.platform in ['win32', 'cygwin']:
-  NPM += '.cmd'
-
 
 def main():
   os.chdir(SOURCE_ROOT)
@@ -65,8 +62,6 @@ def main():
   create_chrome_version_h()
   touch_config_gypi()
   run_update(defines, args.msvs)
-  create_node_headers()
-  update_electron_modules('spec', args.target_arch)
 
 
 def parse_args():
@@ -161,46 +156,6 @@ def setup_libchromiumcontent(is_dev, target_arch, url,
     subprocess.check_call([sys.executable, download, '-s'] + args)
 
 
-def set_clang_env(env):
-  llvm_dir = os.path.join(SOURCE_ROOT, 'vendor', 'llvm-build',
-                          'Release+Asserts', 'bin')
-  env['CC']  = os.path.join(llvm_dir, 'clang')
-  env['CXX'] = os.path.join(llvm_dir, 'clang++')
-
-
-def update_node_modules(dirname, env=None):
-  if env is None:
-    env = os.environ.copy()
-  if PLATFORM == 'linux':
-    # Use prebuilt clang for building native modules.
-    set_clang_env(env)
-    env['npm_config_clang'] = '1'
-  with scoped_cwd(dirname):
-    args = [NPM, 'install']
-    if is_verbose_mode():
-      args += ['--verbose']
-    # Ignore npm install errors when running in CI.
-    if os.environ.has_key('CI'):
-      try:
-        execute_stdout(args, env)
-        execute_stdout([NPM, 'rebuild'], env)
-      except subprocess.CalledProcessError:
-        pass
-    else:
-      execute_stdout(args, env)
-      execute_stdout([NPM, 'rebuild'], env)
-
-
-def update_electron_modules(dirname, target_arch):
-  env = os.environ.copy()
-  version = get_electron_version()
-  env['npm_config_arch']    = target_arch
-  env['npm_config_target']  = version
-  env['npm_config_nodedir'] = os.path.join(SOURCE_ROOT, 'dist',
-                                           'node-{0}'.format(version))
-  update_node_modules(dirname, env)
-
-
 def update_win32_python():
   with scoped_cwd(VENDOR_DIR):
     if not os.path.exists('python_26'):
@@ -271,12 +226,6 @@ def run_update(defines, msvs):
   execute_stdout(args)
 
 
-def create_node_headers():
-  execute_stdout([sys.executable,
-                  os.path.join(SOURCE_ROOT, 'script', 'create-node-headers.py'),
-                  '--version', get_electron_version()])
-
-
 def get_libchromiumcontent_commit():
   commit = os.getenv('LIBCHROMIUMCONTENT_COMMIT')
   if commit:

script/cibuild

@@ -89,7 +89,7 @@ def main():
   else:
     run_script('build.py', ['-c', 'D'])
     if PLATFORM == 'win32' or target_arch == 'x64':
-      run_script('test.py', ['--ci'])
+      run_script('test.py', ['--ci', '--rebuild_native_modules'])
       run_script('verify-ffmpeg.py')
 
 

script/create-node-headers.py

@@ -33,26 +33,34 @@ HEADERS_FILES = [
 
 
 def main():
-  safe_mkdir(DIST_DIR)
-
   args = parse_args()
-  node_headers_dir = os.path.join(DIST_DIR, 'node-{0}'.format(args.version))
+
-  iojs_headers_dir = os.path.join(DIST_DIR, 'iojs-{0}'.format(args.version))
+  safe_mkdir(args.directory)
-  iojs2_headers_dir = os.path.join(DIST_DIR,
+
+  node_headers_dir = os.path.join(args.directory,
+                                  'node-{0}'.format(args.version))
+  iojs_headers_dir = os.path.join(args.directory,
+                                  'iojs-{0}'.format(args.version))
+  iojs2_headers_dir = os.path.join(args.directory,
                                    'iojs-{0}-headers'.format(args.version))
 
   copy_headers(node_headers_dir)
-  create_header_tarball(node_headers_dir)
+  create_header_tarball(args.directory, node_headers_dir)
+
   copy_headers(iojs_headers_dir)
-  create_header_tarball(iojs_headers_dir)
+  create_header_tarball(args.directory, iojs_headers_dir)
+
   copy_headers(iojs2_headers_dir)
-  create_header_tarball(iojs2_headers_dir)
+  create_header_tarball(args.directory, iojs2_headers_dir)
 
 
 def parse_args():
   parser = argparse.ArgumentParser(description='create node header tarballs')
   parser.add_argument('-v', '--version', help='Specify the version',
                       required=True)
+  parser.add_argument('-d', '--directory', help='Specify the output directory',
+                      default=DIST_DIR,
+                      required=False)
   return parser.parse_args()
 
 
@@ -85,9 +93,9 @@ def copy_headers(dist_headers_dir):
                        os.path.join(dist_headers_dir, 'deps'))
 
 
-def create_header_tarball(dist_headers_dir):
+def create_header_tarball(directory, dist_headers_dir):
   target = dist_headers_dir + '.tar.gz'
-  with scoped_cwd(DIST_DIR):
+  with scoped_cwd(directory):
     tarball = tarfile.open(name=target, mode='w:gz')
     tarball.add(os.path.relpath(dist_headers_dir))
     tarball.close()

script/lib/util.py

@@ -15,12 +15,16 @@ import urllib2
 import os
 import zipfile
 
-from config import is_verbose_mode
+from config import is_verbose_mode, PLATFORM
 from env_util import get_vs_env
 
 BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor',
                                         'boto'))
 
+NPM = 'npm'
+if sys.platform in ['win32', 'cygwin']:
+  NPM += '.cmd'
+
 
 def get_host_arch():
   """Returns the host architecture with a predictable string."""
@@ -244,3 +248,42 @@ def import_vs_env(target_arch):
     vs_arch = 'x86_amd64'
   env = get_vs_env('14.0', vs_arch)
   os.environ.update(env)
+
+
+def set_clang_env(env):
+  SOURCE_ROOT = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
+  llvm_dir = os.path.join(SOURCE_ROOT, 'vendor', 'llvm-build',
+                          'Release+Asserts', 'bin')
+  env['CC']  = os.path.join(llvm_dir, 'clang')
+  env['CXX'] = os.path.join(llvm_dir, 'clang++')
+
+
+def update_electron_modules(dirname, target_arch, nodedir):
+  env = os.environ.copy()
+  version = get_electron_version()
+  env['npm_config_arch']    = target_arch
+  env['npm_config_target']  = version
+  env['npm_config_nodedir'] = nodedir
+  update_node_modules(dirname, env)
+  execute_stdout([NPM, 'rebuild'], env, dirname)
+
+
+def update_node_modules(dirname, env=None):
+  if env is None:
+    env = os.environ.copy()
+  if PLATFORM == 'linux':
+    # Use prebuilt clang for building native modules.
+    set_clang_env(env)
+    env['npm_config_clang'] = '1'
+  with scoped_cwd(dirname):
+    args = [NPM, 'install']
+    if is_verbose_mode():
+      args += ['--verbose']
+    # Ignore npm install errors when running in CI.
+    if os.environ.has_key('CI'):
+      try:
+        execute_stdout(args, env)
+      except subprocess.CalledProcessError:
+        pass
+    else:
+      execute_stdout(args, env)

script/rebuild-test-modules.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+from lib.config import PLATFORM, enable_verbose_mode, get_target_arch
+from lib.util import execute_stdout, get_electron_version, safe_mkdir, \
+                     update_node_modules, update_electron_modules
+
+
+SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+
+
+def main():
+  os.chdir(SOURCE_ROOT)
+
+  args = parse_args()
+  config = args.configuration
+
+  if args.verbose:
+    enable_verbose_mode()
+
+  spec_modules = os.path.join(SOURCE_ROOT, 'spec', 'node_modules')
+  out_dir = os.path.join(SOURCE_ROOT, 'out', config)
+  version = get_electron_version()
+  node_dir = os.path.join(out_dir, 'node-{0}'.format(version))
+
+  # Create node headers
+  script_path = os.path.join(SOURCE_ROOT, 'script', 'create-node-headers.py')
+  execute_stdout([sys.executable, script_path, '--version', version,
+                  '--directory', out_dir])
+
+  if PLATFORM == 'win32':
+    lib_dir = os.path.join(node_dir, 'Release')
+    safe_mkdir(lib_dir)
+    iojs_lib = os.path.join(lib_dir, 'iojs.lib')
+    atom_lib = os.path.join(out_dir, 'node.dll.lib')
+    shutil.copy2(atom_lib, iojs_lib)
+
+  # Native modules can only be compiled against release builds on Windows
+  if config == 'R' or PLATFORM != 'win32':
+    update_electron_modules(os.path.dirname(spec_modules), get_target_arch(),
+                            node_dir)
+  else:
+    update_node_modules(os.path.dirname(spec_modules))
+
+
+def parse_args():
+  parser = argparse.ArgumentParser(description='Rebuild native test modules')
+  parser.add_argument('-v', '--verbose',
+                      action='store_true',
+                      help='Prints the output of the subprocesses')
+  parser.add_argument('-c', '--configuration',
+                      help='Build configuration to rebuild modules against',
+                      default='D',
+                      required=False)
+  return parser.parse_args()
+
+
+if __name__ == '__main__':
+  sys.exit(main())

script/test.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python
 
+import argparse
 import os
 import shutil
 import subprocess
 import sys
 
-from lib.util import electron_gyp, rm_rf
+from lib.config import enable_verbose_mode
+from lib.util import electron_gyp, execute_stdout, rm_rf
 
 
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
@@ -17,9 +19,15 @@ PRODUCT_NAME = electron_gyp()['product_name%']
 def main():
   os.chdir(SOURCE_ROOT)
 
-  config = 'D'
+  args = parse_args()
-  if len(sys.argv) == 2 and sys.argv[1] == '-R':
+  config = args.configuration
-    config = 'R'
+
+  if args.verbose:
+    enable_verbose_mode()
+
+  spec_modules = os.path.join(SOURCE_ROOT, 'spec', 'node_modules')
+  if args.rebuild_native_modules or not os.path.isdir(spec_modules):
+    rebuild_native_modules(args.verbose, config)
 
   if sys.platform == 'darwin':
     electron = os.path.join(SOURCE_ROOT, 'out', config,
@@ -36,10 +44,9 @@ def main():
     electron = os.path.join(SOURCE_ROOT, 'out', config, PROJECT_NAME)
     resources_path = os.path.join(SOURCE_ROOT, 'out', config)
 
-  use_instrumented_asar = '--use-instrumented-asar' in sys.argv
   returncode = 0
   try:
-    if use_instrumented_asar:
+    if args.use_instrumented_asar:
       install_instrumented_asar_file(resources_path)
     subprocess.check_call([electron, 'spec'] + sys.argv[1:])
   except subprocess.CalledProcessError as e:
@@ -47,7 +54,7 @@ def main():
   except KeyboardInterrupt:
     returncode = 0
 
-  if use_instrumented_asar:
+  if args.use_instrumented_asar:
     restore_uninstrumented_asar_file(resources_path)
 
   if os.environ.has_key('OUTPUT_TO_FILE'):
@@ -60,6 +67,30 @@ def main():
   return returncode
 
 
+def parse_args():
+  parser = argparse.ArgumentParser(description='Run Electron tests')
+  parser.add_argument('--use_instrumented_asar',
+                      help='Run tests with coverage instructed asar file',
+                      action='store_true',
+                      required=False)
+  parser.add_argument('--rebuild_native_modules',
+                      help='Rebuild native modules used by specs',
+                      action='store_true',
+                      required=False)
+  parser.add_argument('--ci',
+                      help='Run tests in CI mode',
+                      action='store_true',
+                      required=False)
+  parser.add_argument('-v', '--verbose',
+                      action='store_true',
+                      help='Prints the output of the subprocesses')
+  parser.add_argument('-c', '--configuration',
+                      help='Build configuration to run tests against',
+                      default='D',
+                      required=False)
+  return parser.parse_args()
+
+
 def install_instrumented_asar_file(resources_path):
   asar_path = os.path.join(resources_path, '{0}.asar'.format(PROJECT_NAME))
   uninstrumented_path = os.path.join(resources_path,
@@ -78,5 +109,12 @@ def restore_uninstrumented_asar_file(resources_path):
   shutil.move(uninstrumented_path, asar_path)
 
 
+def rebuild_native_modules(verbose, configuration):
+  script_path = os.path.join(SOURCE_ROOT, 'script', 'rebuild-test-modules.py')
+  args = ['--configuration', configuration]
+  if verbose:
+    args += ['--verbose']
+  execute_stdout([sys.executable, script_path] + args)
+
 if __name__ == '__main__':
   sys.exit(main())

spec/api-browser-window-spec.js

@@ -12,6 +12,7 @@ const {ipcRenderer, remote, screen} = require('electron')
 const {app, ipcMain, BrowserWindow, protocol, webContents} = remote
 
 const isCI = remote.getGlobal('isCi')
+const nativeModulesEnabled = remote.getGlobal('nativeModulesEnabled')
 
 describe('BrowserWindow module', function () {
   var fixtures = path.resolve(__dirname, 'fixtures')
@@ -1301,8 +1302,9 @@ describe('BrowserWindow module', function () {
         w.loadURL('file://' + path.join(fixtures, 'api', 'native-window-open-iframe.html'))
       })
 
-      if (process.platform !== 'win32' || process.execPath.toLowerCase().indexOf('\\out\\d\\') === -1) {
       it('loads native addons correctly after reload', (done) => {
+        if (!nativeModulesEnabled) return done()
+
         ipcMain.once('answer', (event, content) => {
           assert.equal(content, 'function')
           ipcMain.once('answer', (event, content) => {
@@ -1313,7 +1315,6 @@ describe('BrowserWindow module', function () {
         })
         w.loadURL('file://' + path.join(fixtures, 'api', 'native-window-open-native-addon.html'))
       })
-      }
     })
   })
 

spec/modules-spec.js

@@ -5,12 +5,15 @@ const {remote} = require('electron')
 const {BrowserWindow} = remote
 const {closeWindow} = require('./window-helpers')
 
+const nativeModulesEnabled = remote.getGlobal('nativeModulesEnabled')
+
 describe('modules support', function () {
   var fixtures = path.join(__dirname, 'fixtures')
 
   describe('third-party module', function () {
-    if (process.platform !== 'win32' || process.execPath.toLowerCase().indexOf('\\out\\d\\') === -1) {
     describe('runas', function () {
+      if (!nativeModulesEnabled) return
+
       it('can be required in renderer', function () {
         require('runas')
       })
@@ -26,6 +29,7 @@ describe('modules support', function () {
     })
 
     describe('ffi', function () {
+      if (!nativeModulesEnabled) return
       if (process.platform === 'win32') return
 
       it('does not crash', function () {
@@ -36,7 +40,6 @@ describe('modules support', function () {
         assert.equal(libm.ceil(1.5), 2)
       })
     })
-    }
 
     describe('q', function () {
       var Q = require('q')

spec/static/main.js

@@ -89,6 +89,8 @@ if (global.isCi) {
   })
 }
 
+global.nativeModulesEnabled = process.platform !== 'win32' || process.execPath.toLowerCase().indexOf('\\out\\d\\') === -1
+
 // Register app as standard scheme.
 global.standardScheme = 'app'
 global.zoomScheme = 'zoom'

spec/webview-spec.js

@@ -7,6 +7,7 @@ const {app, session, getGuestWebContents, ipcMain, BrowserWindow, webContents} =
 const {closeWindow} = require('./window-helpers')
 
 const isCI = remote.getGlobal('isCi')
+const nativeModulesEnabled = remote.getGlobal('nativeModulesEnabled')
 
 describe('<webview> tag', function () {
   this.timeout(3 * 60 * 1000)
@@ -171,8 +172,9 @@ describe('<webview> tag', function () {
       document.body.appendChild(webview)
     })
 
-    if (process.platform !== 'win32' || process.execPath.toLowerCase().indexOf('\\out\\d\\') === -1) {
     it('loads native modules when navigation happens', function (done) {
+      if (!nativeModulesEnabled) return done()
+
       var listener = function () {
         webview.removeEventListener('did-finish-load', listener)
         var listener2 = function (e) {
@@ -187,7 +189,6 @@ describe('<webview> tag', function () {
       webview.src = 'file://' + fixtures + '/pages/native-module.html'
       document.body.appendChild(webview)
     })
-    }
   })
 
   describe('preload attribute', function () {