diff --git a/.circleci/config.yml b/.circleci/config.yml index c738a517a88..ce7297522cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,7 @@ env-testing-build: &env-testing-build env-release-build: &env-release-build GN_CONFIG: //electron/build/args/release.gn NOTIFY_SLACK: true + ELECTRON_RELEASE: 1 # Build targets options. env-ia32: &env-ia32 @@ -226,6 +227,18 @@ step-mksnapshot-store: &step-mksnapshot-store path: src/out/Default/mksnapshot.zip destination: mksnapshot.zip +step-maybe-generate-breakpad_symbols: &step-maybe-generate-breakpad_symbols + run: + name: Generate breakpad symbols + command: | + if [ "$ELECTRON_RELEASE" == "1" ]; then + cd src + # Build needed dump_syms executable + ninja -C out/Default third_party/breakpad:dump_syms + electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" + fi + + # Lists of steps. steps-checkout: &steps-checkout steps: @@ -288,6 +301,7 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests - *step-electron-build - *step-electron-dist-build - *step-electron-dist-store + - *step-maybe-generate-breakpad_symbols # mksnapshot - *step-mksnapshot-build diff --git a/appveyor.yml b/appveyor.yml index 3e8fd8c9200..d0871e5cdec 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -33,7 +33,13 @@ build_script: - ninja -C out/Default electron:electron_chromedriver_zip - appveyor PushArtifact out/Default/dist.zip - appveyor PushArtifact out/Default/chromedriver.zip + - appveyor PushArtifact out/ffmpeg/ffmpeg.zip + - ps: >- + if ($env:GN_CONFIG -eq 'release') { + python electron\script\dump-symbols.py -d %cd%/out/Default/electron.breakpad.syms + appveyor PushArtifact out/Default/electron.breakpad.syms + } test_script: - if "%GN_CONFIG%"=="testing" ( echo Verifying non proprietary ffmpeg & python electron\script\verify-ffmpeg.py --build-dir out\Default --source-root %cd% --ffmpeg-path out\ffmpeg ) - ps: >- diff --git a/script/dump-symbols.py b/script/dump-symbols.py index 8d62db3fc57..963b07773aa 100755 --- a/script/dump-symbols.py +++ b/script/dump-symbols.py @@ -1,62 +1,88 @@ #!/usr/bin/env python +import argparse import os import sys -from lib.config import PLATFORM -from lib.util import electron_gyp, execute, rm_rf +from lib.config import PLATFORM, enable_verbose_mode, is_verbose_mode +from lib.gn import gn +from lib.util import execute, rm_rf +ELECTRON_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +SOURCE_ROOT = os.path.abspath(os.path.dirname(ELECTRON_ROOT)) +RELEASE_PATH = os.path.join('out', 'Release') -SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') -OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R') -CHROMIUM_DIR = os.path.join(SOURCE_ROOT, 'vendor', 'download', - 'libchromiumcontent', 'static_library') - - -def main(destination): - rm_rf(destination) - (project_name, product_name) = get_names_from_gyp() +def main(): + args = parse_args() + if args.verbose: + enable_verbose_mode() + rm_rf(args.destination) + source_root = os.path.abspath(args.source_root) + build_path = os.path.join(source_root, args.build_dir) + product_name = gn(build_path).args().get_string('electron_product_name') + project_name = gn(build_path).args().get_string('electron_project_name') if PLATFORM in ['darwin', 'linux']: - generate_breakpad_symbols = os.path.join(SOURCE_ROOT, 'tools', 'posix', - 'generate_breakpad_symbols.py') + if PLATFORM == 'darwin': #macOS has an additional helper app; provide the path to that binary also - main_app = os.path.join(OUT_DIR, '{0}.app'.format(product_name), + main_app = os.path.join(build_path, '{0}.app'.format(product_name), 'Contents', 'MacOS', product_name) helper_name = product_name + " Helper" - helper_app = os.path.join(OUT_DIR, '{0}.app'.format(helper_name), + helper_app = os.path.join(build_path, '{0}.app'.format(helper_name), 'Contents', 'MacOS', product_name + " Helper") binaries = [main_app, helper_app] + for binary in binaries: + generate_posix_symbols(binary, source_root, build_path, + args.destination) else: - binaries = [os.path.join(OUT_DIR, project_name)] - args = [ - '--build-dir={0}'.format(OUT_DIR), - '--symbols-dir={0}'.format(destination), - '--libchromiumcontent-dir={0}'.format(CHROMIUM_DIR), - '--clear', - '--jobs=16', - ] - for binary in binaries: - args += '--binary={0}'.format(binary), + binary = os.path.join(build_path, project_name) + generate_posix_symbols(binary, source_root, build_path, + args.destination) else: - generate_breakpad_symbols = os.path.join(SOURCE_ROOT, 'tools', 'win', + generate_breakpad_symbols = os.path.join(ELECTRON_ROOT, 'tools', 'win', 'generate_breakpad_symbols.py') args = [ - '--symbols-dir={0}'.format(destination), + '--symbols-dir={0}'.format(args.destination), '--jobs=16', - os.path.relpath(OUT_DIR), - ] + os.path.relpath(build_path), + ] + execute([sys.executable, generate_breakpad_symbols] + args) - execute([sys.executable, generate_breakpad_symbols] + args) - - -def get_names_from_gyp(): - variables = electron_gyp() - return (variables['project_name%'], variables['product_name%']) +def generate_posix_symbols(binary, source_root, build_dir, destination): + generate_breakpad_symbols = os.path.join(source_root, 'components', 'crash', + 'content', 'tools', + 'generate_breakpad_symbols.py') + args = [ + '--build-dir={0}'.format(build_dir), + '--symbols-dir={0}'.format(destination), + '--jobs=16', + '--binary={0}'.format(binary), + ] + if is_verbose_mode(): + args += ['-v'] + execute([sys.executable, generate_breakpad_symbols] + args) +def parse_args(): + parser = argparse.ArgumentParser(description='Create breakpad symbols') + parser.add_argument('-b', '--build-dir', + help='Path to an Electron build folder. \ + Relative to the --source-root.', + default=RELEASE_PATH, + required=False) + parser.add_argument('-d', '--destination', + help='Path to save symbols to.', + default=None, + required=True) + parser.add_argument('-s', '--source-root', + help='Path to the src folder.', + default=SOURCE_ROOT, + required=False) + parser.add_argument('-v', '--verbose', + action='store_true', + help='Prints the output of the subprocesses') + return parser.parse_args() if __name__ == '__main__': - sys.exit(main(sys.argv[1])) + sys.exit(main()) diff --git a/tools/posix/generate_breakpad_symbols.py b/tools/posix/generate_breakpad_symbols.py deleted file mode 100755 index 8d6fb598cc1..00000000000 --- a/tools/posix/generate_breakpad_symbols.py +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2013 GitHub, Inc. -# Copyright (c) 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""A tool to generate symbols for a binary suitable for breakpad. - -Currently, the tool only supports Linux, Android, and Mac. Support for other -platforms is planned. -""" - -import errno -import argparse -import os -import Queue -import re -import shutil -import subprocess -import sys -import threading - - -CONCURRENT_TASKS=4 - - -def GetCommandOutput(command): - """Runs the command list, returning its output. - - Prints the given command (which should be a list of one or more strings), - then runs it and returns its output (stdout) as a string. - - From chromium_utils. - """ - devnull = open(os.devnull, 'w') - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=devnull, - bufsize=1) - output = proc.communicate()[0] - return output - - -def GetDumpSymsBinary(build_dir=None): - """Returns the path to the dump_syms binary.""" - DUMP_SYMS = 'dump_syms' - dump_syms_bin = os.path.join(os.path.expanduser(build_dir), DUMP_SYMS) - if not os.access(dump_syms_bin, os.X_OK): - print 'Cannot find %s.' % DUMP_SYMS - sys.exit(1) - - return dump_syms_bin - - -def FindBundlePart(full_path): - if full_path.endswith(('.dylib', '.framework', '.app')): - return os.path.basename(full_path) - elif full_path != '' and full_path != '/': - return FindBundlePart(os.path.dirname(full_path)) - else: - return '' - - -def GetDSYMBundle(options, binary_path): - """Finds the .dSYM bundle to the binary.""" - if os.path.isabs(binary_path): - dsym_path = binary_path + '.dSYM' - if os.path.exists(dsym_path): - return dsym_path - - filename = FindBundlePart(binary_path) - search_dirs = [options.build_dir, options.libchromiumcontent_dir] - if filename.endswith(('.dylib', '.framework', '.app')): - for directory in search_dirs: - dsym_path = os.path.join(directory, filename) + '.dSYM' - if os.path.exists(dsym_path): - return dsym_path - - return binary_path - - -def GetSymbolPath(options, binary_path): - """Finds the .dbg to the binary.""" - filename = os.path.basename(binary_path) - dbg_path = os.path.join(options.libchromiumcontent_dir, filename) + '.dbg' - if os.path.exists(dbg_path): - return dbg_path - - return binary_path - - -def Resolve(path, exe_path, loader_path, rpaths): - """Resolve a dyld path. - - @executable_path is replaced with |exe_path| - @loader_path is replaced with |loader_path| - @rpath is replaced with the first path in |rpaths| where the referenced file - is found - """ - path = path.replace('@loader_path', loader_path) - path = path.replace('@executable_path', exe_path) - if path.find('@rpath') != -1: - for rpath in rpaths: - new_path = Resolve(path.replace('@rpath', rpath), exe_path, loader_path, - []) - if os.access(new_path, os.F_OK): - return new_path - return '' - return path - - -def GetSharedLibraryDependenciesLinux(binary): - """Return absolute paths to all shared library dependecies of the binary. - - This implementation assumes that we're running on a Linux system.""" - ldd = GetCommandOutput(['ldd', binary]) - lib_re = re.compile('\t.* => (.+) \(.*\)$') - result = [] - for line in ldd.splitlines(): - m = lib_re.match(line) - if m: - result.append(os.path.realpath(m.group(1))) - return result - - -def GetSharedLibraryDependenciesMac(binary, exe_path): - """Return absolute paths to all shared library dependecies of the binary. - - This implementation assumes that we're running on a Mac system.""" - loader_path = os.path.dirname(binary) - otool = GetCommandOutput(['otool', '-l', binary]).splitlines() - rpaths = [] - for idx, line in enumerate(otool): - if line.find('cmd LC_RPATH') != -1: - m = re.match(' *path (.*) \(offset .*\)$', otool[idx+2]) - rpaths.append(m.group(1)) - - otool = GetCommandOutput(['otool', '-L', binary]).splitlines() - lib_re = re.compile('\t(.*) \(compatibility .*\)$') - deps = [] - for line in otool: - m = lib_re.match(line) - if m: - dep = Resolve(m.group(1), exe_path, loader_path, rpaths) - if dep: - deps.append(os.path.normpath(dep)) - return deps - - -def GetSharedLibraryDependencies(options, binary, exe_path): - """Return absolute paths to all shared library dependecies of the binary.""" - deps = [] - if sys.platform.startswith('linux'): - deps = GetSharedLibraryDependenciesLinux(binary) - elif sys.platform == 'darwin': - deps = GetSharedLibraryDependenciesMac(binary, exe_path) - else: - print "Platform not supported." - sys.exit(1) - - result = [] - build_dir = os.path.abspath(options.build_dir) - for dep in deps: - if (os.access(dep, os.F_OK)): - result.append(dep) - return result - - -def mkdir_p(path): - """Simulates mkdir -p.""" - try: - os.makedirs(path) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(path): - pass - else: raise - - -def GenerateSymbols(options, binaries): - """Dumps the symbols of binary and places them in the given directory.""" - - queue = Queue.Queue() - print_lock = threading.Lock() - - def _Worker(): - while True: - binary = queue.get() - - if options.verbose: - with print_lock: - print "Generating symbols for %s" % binary - - if sys.platform == 'darwin': - binary = GetDSYMBundle(options, binary) - elif sys.platform == 'linux2': - binary = GetSymbolPath(options, binary) - - syms = GetCommandOutput([GetDumpSymsBinary(options.build_dir), '-r', '-c', - binary]) - module_line = re.match("MODULE [^ ]+ [^ ]+ ([0-9A-F]+) (.*)\n", syms) - output_path = os.path.join(options.symbols_dir, module_line.group(2), - module_line.group(1)) - mkdir_p(output_path) - symbol_file = "%s.sym" % module_line.group(2) - f = open(os.path.join(output_path, symbol_file), 'w') - f.write(syms) - f.close() - - queue.task_done() - - for binary in binaries: - queue.put(binary) - - for _ in range(options.jobs): - t = threading.Thread(target=_Worker) - t.daemon = True - t.start() - - queue.join() - - -def main(): - parser = argparse.ArgumentParser(description='Generate Breakpad Symbols Project') - parser.add_argument('--build-dir', required=True, - help='The build output directory.') - parser.add_argument('--symbols-dir', required=True, - help='The directory where to write the symbols file.') - parser.add_argument('--libchromiumcontent-dir', required=True, - help='The directory where libchromiumcontent is downloaded.') - parser.add_argument('--binary', action='append', required=True, - help='The path of the binary to generate symbols for.') - parser.add_argument('--clear', default=False, action='store_true', - help='Clear the symbols directory before writing new ' - 'symbols.') - parser.add_argument('-j', '--jobs', default=CONCURRENT_TASKS, action='store', - type=int, help='Number of parallel tasks to run.') - parser.add_argument('-v', '--verbose', action='store_true', - help='Print verbose status output.') - - options = parser.parse_args() - - for bin_file in options.binary: - if not os.access(bin_file, os.X_OK): - print "Cannot find %s." % options.binary - return 1 - - if options.clear: - try: - shutil.rmtree(options.symbols_dir) - except: - pass - - # Build the transitive closure of all dependencies. - binaries = set(options.binary) - queue = options.binary - while queue: - current_bin = queue.pop(0) - exe_path = os.path.dirname(current_bin) - deps = GetSharedLibraryDependencies(options, current_bin, exe_path) - new_deps = set(deps) - binaries - binaries |= new_deps - queue.extend(list(new_deps)) - - GenerateSymbols(options, binaries) - - return 0 - - -if '__main__' == __name__: - sys.exit(main()) diff --git a/tools/posix/strip.sh b/tools/posix/strip.sh deleted file mode 100755 index 4f4f19ae66e..00000000000 --- a/tools/posix/strip.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Ignore errors from strip. -strip "$@" - -exit 0 diff --git a/vsts.yml b/vsts.yml index 2e72426fbbb..69a988cdbf9 100644 --- a/vsts.yml +++ b/vsts.yml @@ -65,6 +65,14 @@ jobs: displayName: Non proprietary ffmpeg build condition: and(succeeded(), eq(variables['RUN_TESTS'], '1')) + - bash: | + cd src + # Build needed dump_syms executable + ninja -C out/Default third_party/breakpad:dump_syms + electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" + displayName: Ninja build app + condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1')) + - bash: | "$SCCACHE_BINARY" --stop-server displayName: Check sccache stats after build