build: Generate breakpad symbols for GN release builds (#14791)

* Use chromium 's generate_breakpad_symbols.py

* Add breakpad symbol gen to CI.

.circleci/config.yml

@@ -25,6 +25,7 @@ env-testing-build: &env-testing-build
 env-release-build: &env-release-build
   GN_CONFIG: //electron/build/args/release.gn
   NOTIFY_SLACK: true
+  ELECTRON_RELEASE: 1
 
 # Build targets options.
 env-ia32: &env-ia32
@@ -226,6 +227,18 @@ step-mksnapshot-store: &step-mksnapshot-store
     path: src/out/Default/mksnapshot.zip
     destination: mksnapshot.zip
 
+step-maybe-generate-breakpad_symbols: &step-maybe-generate-breakpad_symbols
+  run:
+    name: Generate breakpad symbols
+    command: |
+      if [ "$ELECTRON_RELEASE" == "1" ]; then
+        cd src
+        # Build needed dump_syms executable
+        ninja -C out/Default third_party/breakpad:dump_syms
+        electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms"
+      fi
+
+
 # Lists of steps.
 steps-checkout: &steps-checkout
   steps:
@@ -288,6 +301,7 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests
     - *step-electron-build
     - *step-electron-dist-build
     - *step-electron-dist-store
+    - *step-maybe-generate-breakpad_symbols
 
     # mksnapshot
     - *step-mksnapshot-build

appveyor.yml

@@ -33,7 +33,13 @@ build_script:
   - ninja -C out/Default electron:electron_chromedriver_zip
   - appveyor PushArtifact out/Default/dist.zip
   - appveyor PushArtifact out/Default/chromedriver.zip
+
   - appveyor PushArtifact out/ffmpeg/ffmpeg.zip
+  - ps: >-
+      if ($env:GN_CONFIG -eq 'release') {
+        python electron\script\dump-symbols.py -d %cd%/out/Default/electron.breakpad.syms
+        appveyor PushArtifact out/Default/electron.breakpad.syms
+      }
 test_script:
   - if "%GN_CONFIG%"=="testing" ( echo Verifying non proprietary ffmpeg & python electron\script\verify-ffmpeg.py --build-dir out\Default --source-root %cd% --ffmpeg-path out\ffmpeg )
   - ps: >-

script/dump-symbols.py

@@ -1,62 +1,88 @@
 #!/usr/bin/env python
 
+import argparse
 import os
 import sys
 
-from lib.config import PLATFORM
+from lib.config import PLATFORM, enable_verbose_mode, is_verbose_mode
-from lib.util import electron_gyp, execute, rm_rf
+from lib.gn import gn
+from lib.util import execute, rm_rf
 
+ELECTRON_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+SOURCE_ROOT = os.path.abspath(os.path.dirname(ELECTRON_ROOT))
+RELEASE_PATH = os.path.join('out', 'Release')
 
-SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+def main():
-DIST_DIR = os.path.join(SOURCE_ROOT, 'dist')
+  args = parse_args()
-OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R')
+  if args.verbose:
-CHROMIUM_DIR = os.path.join(SOURCE_ROOT, 'vendor', 'download',
+    enable_verbose_mode()
-                            'libchromiumcontent', 'static_library')
+  rm_rf(args.destination)
-
+  source_root = os.path.abspath(args.source_root)
-
+  build_path = os.path.join(source_root, args.build_dir)
-def main(destination):
+  product_name = gn(build_path).args().get_string('electron_product_name')
-  rm_rf(destination)
+  project_name = gn(build_path).args().get_string('electron_project_name')
-  (project_name, product_name) = get_names_from_gyp()
 
   if PLATFORM in ['darwin', 'linux']:
-    generate_breakpad_symbols = os.path.join(SOURCE_ROOT, 'tools', 'posix',
+
-                                             'generate_breakpad_symbols.py')
     if PLATFORM == 'darwin':
       #macOS has an additional helper app; provide the path to that binary also
-      main_app = os.path.join(OUT_DIR, '{0}.app'.format(product_name),
+      main_app = os.path.join(build_path, '{0}.app'.format(product_name),
                             'Contents', 'MacOS', product_name)
       helper_name = product_name + " Helper"
-      helper_app = os.path.join(OUT_DIR, '{0}.app'.format(helper_name),
+      helper_app = os.path.join(build_path, '{0}.app'.format(helper_name),
                             'Contents', 'MacOS', product_name + " Helper")
       binaries = [main_app, helper_app]
-    else:
-      binaries = [os.path.join(OUT_DIR, project_name)]
-    args = [
-      '--build-dir={0}'.format(OUT_DIR),
-      '--symbols-dir={0}'.format(destination),
-      '--libchromiumcontent-dir={0}'.format(CHROMIUM_DIR),
-      '--clear',
-      '--jobs=16',
-    ]
       for binary in binaries:        
-      args += '--binary={0}'.format(binary),
+        generate_posix_symbols(binary, source_root, build_path, 
+                                        args.destination)
+    else:
+      binary = os.path.join(build_path, project_name)
+      generate_posix_symbols(binary, source_root, build_path, 
+                                      args.destination)
 
   else:
-    generate_breakpad_symbols = os.path.join(SOURCE_ROOT, 'tools', 'win',
+    generate_breakpad_symbols = os.path.join(ELECTRON_ROOT, 'tools', 'win',
                                              'generate_breakpad_symbols.py')
     args = [
-      '--symbols-dir={0}'.format(destination),
+      '--symbols-dir={0}'.format(args.destination),
       '--jobs=16',
-      os.path.relpath(OUT_DIR),
+      os.path.relpath(build_path),
     ]  
-
     execute([sys.executable, generate_breakpad_symbols] + args)
 
+def generate_posix_symbols(binary, source_root, build_dir, destination):
+  generate_breakpad_symbols = os.path.join(source_root, 'components', 'crash',
+                                          'content', 'tools',
+                                          'generate_breakpad_symbols.py')
+  args = [
+    '--build-dir={0}'.format(build_dir),
+    '--symbols-dir={0}'.format(destination),
+    '--jobs=16',
+    '--binary={0}'.format(binary),
+  ]
+  if is_verbose_mode(): 
+    args += ['-v']
+  execute([sys.executable, generate_breakpad_symbols] + args)    
 
-def get_names_from_gyp():
+def parse_args():
-  variables = electron_gyp()
+  parser = argparse.ArgumentParser(description='Create breakpad symbols')
-  return (variables['project_name%'], variables['product_name%'])
+  parser.add_argument('-b', '--build-dir',
-
+                      help='Path to an Electron build folder. \
+                          Relative to the --source-root.',
+                      default=RELEASE_PATH,
+                      required=False)
+  parser.add_argument('-d', '--destination',
+                      help='Path to save symbols to.',
+                      default=None,
+                      required=True)
+  parser.add_argument('-s', '--source-root',
+                      help='Path to the src folder.',
+                      default=SOURCE_ROOT,
+                      required=False)
+  parser.add_argument('-v', '--verbose',
+                      action='store_true',
+                      help='Prints the output of the subprocesses')                      
+  return parser.parse_args()
 
 if __name__ == '__main__':
-  sys.exit(main(sys.argv[1]))
+  sys.exit(main())

tools/posix/generate_breakpad_symbols.py

@@ -1,268 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2013 GitHub, Inc.
-# Copyright (c) 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A tool to generate symbols for a binary suitable for breakpad.
-
-Currently, the tool only supports Linux, Android, and Mac. Support for other
-platforms is planned.
-"""
-
-import errno
-import argparse
-import os
-import Queue
-import re
-import shutil
-import subprocess
-import sys
-import threading
-
-
-CONCURRENT_TASKS=4
-
-
-def GetCommandOutput(command):
-  """Runs the command list, returning its output.
-
-  Prints the given command (which should be a list of one or more strings),
-  then runs it and returns its output (stdout) as a string.
-
-  From chromium_utils.
-  """
-  devnull = open(os.devnull, 'w')
-  proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=devnull,
-                          bufsize=1)
-  output = proc.communicate()[0]
-  return output
-
-
-def GetDumpSymsBinary(build_dir=None):
-  """Returns the path to the dump_syms binary."""
-  DUMP_SYMS = 'dump_syms'
-  dump_syms_bin = os.path.join(os.path.expanduser(build_dir), DUMP_SYMS)
-  if not os.access(dump_syms_bin, os.X_OK):
-    print 'Cannot find %s.' % DUMP_SYMS
-    sys.exit(1)
-
-  return dump_syms_bin
-
-
-def FindBundlePart(full_path):
-  if full_path.endswith(('.dylib', '.framework', '.app')):
-    return os.path.basename(full_path)
-  elif full_path != '' and full_path != '/':
-    return FindBundlePart(os.path.dirname(full_path))
-  else:
-    return ''
-
-
-def GetDSYMBundle(options, binary_path):
-  """Finds the .dSYM bundle to the binary."""
-  if os.path.isabs(binary_path):
-    dsym_path = binary_path + '.dSYM'
-    if os.path.exists(dsym_path):
-      return dsym_path
-
-  filename = FindBundlePart(binary_path)
-  search_dirs = [options.build_dir, options.libchromiumcontent_dir]
-  if filename.endswith(('.dylib', '.framework', '.app')):
-    for directory in search_dirs:
-      dsym_path = os.path.join(directory, filename) + '.dSYM'
-      if os.path.exists(dsym_path):
-        return dsym_path
-
-  return binary_path
-
-
-def GetSymbolPath(options, binary_path):
-  """Finds the .dbg to the binary."""
-  filename = os.path.basename(binary_path)
-  dbg_path = os.path.join(options.libchromiumcontent_dir, filename) + '.dbg'
-  if os.path.exists(dbg_path):
-    return dbg_path
-
-  return binary_path
-
-
-def Resolve(path, exe_path, loader_path, rpaths):
-  """Resolve a dyld path.
-
-  @executable_path is replaced with |exe_path|
-  @loader_path is replaced with |loader_path|
-  @rpath is replaced with the first path in |rpaths| where the referenced file
-      is found
-  """
-  path = path.replace('@loader_path', loader_path)
-  path = path.replace('@executable_path', exe_path)
-  if path.find('@rpath') != -1:
-    for rpath in rpaths:
-      new_path = Resolve(path.replace('@rpath', rpath), exe_path, loader_path,
-                         [])
-      if os.access(new_path, os.F_OK):
-        return new_path
-    return ''
-  return path
-
-
-def GetSharedLibraryDependenciesLinux(binary):
-  """Return absolute paths to all shared library dependecies of the binary.
-
-  This implementation assumes that we're running on a Linux system."""
-  ldd = GetCommandOutput(['ldd', binary])
-  lib_re = re.compile('\t.* => (.+) \(.*\)$')
-  result = []
-  for line in ldd.splitlines():
-    m = lib_re.match(line)
-    if m:
-      result.append(os.path.realpath(m.group(1)))
-  return result
-
-
-def GetSharedLibraryDependenciesMac(binary, exe_path):
-  """Return absolute paths to all shared library dependecies of the binary.
-
-  This implementation assumes that we're running on a Mac system."""
-  loader_path = os.path.dirname(binary)
-  otool = GetCommandOutput(['otool', '-l', binary]).splitlines()
-  rpaths = []
-  for idx, line in enumerate(otool):
-    if line.find('cmd LC_RPATH') != -1:
-      m = re.match(' *path (.*) \(offset .*\)$', otool[idx+2])
-      rpaths.append(m.group(1))
-
-  otool = GetCommandOutput(['otool', '-L', binary]).splitlines()
-  lib_re = re.compile('\t(.*) \(compatibility .*\)$')
-  deps = []
-  for line in otool:
-    m = lib_re.match(line)
-    if m:
-      dep = Resolve(m.group(1), exe_path, loader_path, rpaths)
-      if dep:
-        deps.append(os.path.normpath(dep))
-  return deps
-
-
-def GetSharedLibraryDependencies(options, binary, exe_path):
-  """Return absolute paths to all shared library dependecies of the binary."""
-  deps = []
-  if sys.platform.startswith('linux'):
-    deps = GetSharedLibraryDependenciesLinux(binary)
-  elif sys.platform == 'darwin':
-    deps = GetSharedLibraryDependenciesMac(binary, exe_path)
-  else:
-    print "Platform not supported."
-    sys.exit(1)
-
-  result = []
-  build_dir = os.path.abspath(options.build_dir)
-  for dep in deps:
-    if (os.access(dep, os.F_OK)):
-      result.append(dep)
-  return result
-
-
-def mkdir_p(path):
-  """Simulates mkdir -p."""
-  try:
-    os.makedirs(path)
-  except OSError as e:
-    if e.errno == errno.EEXIST and os.path.isdir(path):
-      pass
-    else: raise
-
-
-def GenerateSymbols(options, binaries):
-  """Dumps the symbols of binary and places them in the given directory."""
-
-  queue = Queue.Queue()
-  print_lock = threading.Lock()
-
-  def _Worker():
-    while True:
-      binary = queue.get()
-
-      if options.verbose:
-        with print_lock:
-          print "Generating symbols for %s" % binary
-
-      if sys.platform == 'darwin':
-        binary = GetDSYMBundle(options, binary)
-      elif sys.platform == 'linux2':
-        binary = GetSymbolPath(options, binary)
-
-      syms = GetCommandOutput([GetDumpSymsBinary(options.build_dir), '-r', '-c',
-                               binary])
-      module_line = re.match("MODULE [^ ]+ [^ ]+ ([0-9A-F]+) (.*)\n", syms)
-      output_path = os.path.join(options.symbols_dir, module_line.group(2),
-                                 module_line.group(1))
-      mkdir_p(output_path)
-      symbol_file = "%s.sym" % module_line.group(2)
-      f = open(os.path.join(output_path, symbol_file), 'w')
-      f.write(syms)
-      f.close()
-
-      queue.task_done()
-
-  for binary in binaries:
-    queue.put(binary)
-
-  for _ in range(options.jobs):
-    t = threading.Thread(target=_Worker)
-    t.daemon = True
-    t.start()
-
-  queue.join()
-
-
-def main():
-  parser = argparse.ArgumentParser(description='Generate Breakpad Symbols Project')
-  parser.add_argument('--build-dir', required=True,
-                    help='The build output directory.')
-  parser.add_argument('--symbols-dir', required=True,
-                    help='The directory where to write the symbols file.')
-  parser.add_argument('--libchromiumcontent-dir', required=True,
-                    help='The directory where libchromiumcontent is downloaded.')
-  parser.add_argument('--binary', action='append', required=True,
-                    help='The path of the binary to generate symbols for.')
-  parser.add_argument('--clear', default=False, action='store_true',
-                    help='Clear the symbols directory before writing new '
-                         'symbols.')
-  parser.add_argument('-j', '--jobs', default=CONCURRENT_TASKS, action='store',
-                    type=int, help='Number of parallel tasks to run.')
-  parser.add_argument('-v', '--verbose', action='store_true',
-                    help='Print verbose status output.')
-
-  options = parser.parse_args()
-
-  for bin_file in options.binary:
-    if not os.access(bin_file, os.X_OK):
-      print "Cannot find %s." % options.binary
-      return 1
-
-  if options.clear:
-    try:
-      shutil.rmtree(options.symbols_dir)
-    except:
-      pass
-
-  # Build the transitive closure of all dependencies.
-  binaries = set(options.binary)
-  queue = options.binary
-  while queue:
-    current_bin = queue.pop(0)
-    exe_path = os.path.dirname(current_bin)
-    deps = GetSharedLibraryDependencies(options, current_bin, exe_path)
-    new_deps = set(deps) - binaries
-    binaries |= new_deps
-    queue.extend(list(new_deps))
-
-  GenerateSymbols(options, binaries)
-
-  return 0
-
-
-if '__main__' == __name__:
-  sys.exit(main())

tools/posix/strip.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-# Ignore errors from strip.
-strip "$@"
-
-exit 0

vsts.yml

@@ -65,6 +65,14 @@ jobs:
     displayName: Non proprietary ffmpeg build
     condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
 
+  - bash: |
+      cd src
+      # Build needed dump_syms executable
+      ninja -C out/Default third_party/breakpad:dump_syms
+      electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms"
+    displayName: Ninja build app
+    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
+
   - bash: |
       "$SCCACHE_BINARY" --stop-server
     displayName: Check sccache stats after build