From f82f89b2a339bb42d12aa50ed7abe5c53836d9d3 Mon Sep 17 00:00:00 2001 From: Jeremy Apthorp Date: Tue, 21 Aug 2018 10:06:28 -0700 Subject: [PATCH] build: [gn] add distributable zip target (#14093) * build: [gn] add distributable zip target * build: update the CircleCI config - enable debug and testing builds on Mac - run Mac release builds nightly - run test for Mac release builds - use shared build machines configs * Add resources dir to zip on non mac platforms. --- .circleci/config.yml | 188 +++++++++++++++++++++++++++++++++-------- BUILD.gn | 73 ++++++++++++++-- appveyor-gn.yml | 10 ++- build/args/release.gn | 1 + build/write_version.py | 11 +++ build/zip.py | 59 +++++++++++++ vsts-gn.yml | 20 ++++- 7 files changed, 316 insertions(+), 46 deletions(-) create mode 100644 build/write_version.py create mode 100644 build/zip.py diff --git a/.circleci/config.yml b/.circleci/config.yml index d66102fb9530..716cf849e9fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -192,6 +192,11 @@ gn-build-steps: &gn-build-steps command: | cd src ninja -C out/Default electron:electron_app + - run: + name: Build dist.zip + command: | + cd src + ninja -C out/Default electron:electron_dist_zip - run: name: Check sccache stats after build command: $SCCACHE_WRAPPER -s @@ -217,6 +222,88 @@ gn-build-steps: &gn-build-steps path: src/junit - store_artifacts: path: src/junit + - store_artifacts: + path: src/out/Default/dist.zip + +gn-mac-build-steps: &gn-mac-build-steps + steps: + - run: + name: Setup depot tools + command: | + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git + echo 'export PATH="$PATH:'"$PWD"'/depot_tools"' >> $BASH_ENV + echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV + - run: + name: Install Node.js 10 on MacOS + command: | + echo 'Installing Node.js 10 for MacOS' + brew update + brew install node@10 + brew install gnu-tar + echo 'export PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"' >> $BASH_ENV + - checkout: + path: src/electron + - run: + name: Gclient sync + command: | + gclient config \ + --name "src/electron" \ + --unmanaged \ + $GCLIENT_EXTRA_ARGS \ + "$CIRCLE_REPOSITORY_URL" + gclient sync --with_branch_heads --with_tags + - run: + name: GN gen + command: | + cd src + SCCACHE_PATH="$PWD/libchromiumcontent/tools/sccache/aad2120/mac/sccache" + echo 'export SCCACHE_WRAPPER="'"$SCCACHE_PATH"'"' >> $BASH_ENV + echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/buildtools"' >> $BASH_ENV + source $BASH_ENV + gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + - run: + name: Ninja build + command: | + cd src + ninja -C out/Default electron:electron_app + - run: + name: Build dist.zip + command: | + cd src + ninja -C out/Default electron:electron_dist_zip + - run: + name: Check sccache stats after build + command: $SCCACHE_WRAPPER -s + - run: + name: Test + environment: + MOCHA_REPORTER: mocha-junit-reporter + MOCHA_FILE: junit/test-results.xml + ELECTRON_DISABLE_SECURITY_WARNINGS: 1 + command: | + if [ "$RUN_TESTS" != "false" ]; then + cd src + ninja -C out/Default third_party/electron_node:headers + export npm_config_nodedir="$PWD/out/Default/gen/node_headers" + (cd electron/spec && npm install) + ./out/Default/Electron.app/Contents/MacOS/Electron electron/spec --ci --enable-logging + fi + - store_test_results: + path: src/junit + - store_artifacts: + path: src/junit + - store_artifacts: + path: src/out/Default/dist.zip + +gn-linux-build-machine: &gn-linux-build-machine + docker: + - image: electronbuilds/electron:0.0.8 + resource_class: 2xlarge + +gn-mac-build-machine: &gn-mac-build-machine + macos: + xcode: "8.3.3" + resource_class: large build-defaults: &build-defaults docker: @@ -389,27 +476,21 @@ jobs: DISPLAY: ':99.0' GN_CONFIG: //electron/build/args/debug.gn RUN_TESTS: false - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-x64-testing-fyi: environment: DISPLAY: ':99.0' GN_CONFIG: //electron/build/args/testing.gn - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-x64-release-fyi: environment: DISPLAY: ':99.0' GN_CONFIG: //electron/build/args/release.gn - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-ia32-debug-fyi: @@ -419,9 +500,7 @@ jobs: GN_EXTRA_ARGS: 'target_cpu = "x86"' NPM_CONFIG_ARCH: ia32 RUN_TESTS: false - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-ia32-testing-fyi: @@ -430,9 +509,7 @@ jobs: GN_CONFIG: //electron/build/args/testing.gn GN_EXTRA_ARGS: 'target_cpu = "x86"' NPM_CONFIG_ARCH: ia32 - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-ia32-release-fyi: @@ -441,9 +518,7 @@ jobs: GN_CONFIG: //electron/build/args/release.gn GN_EXTRA_ARGS: 'target_cpu = "x86"' NPM_CONFIG_ARCH: ia32 - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-arm-debug-fyi: @@ -452,9 +527,7 @@ jobs: GN_EXTRA_ARGS: 'target_cpu = "arm"' GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True' RUN_TESTS: false - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-arm-testing-fyi: @@ -463,9 +536,7 @@ jobs: GN_EXTRA_ARGS: 'target_cpu = "arm"' GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True' RUN_TESTS: false - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-arm-release-fyi: @@ -474,9 +545,7 @@ jobs: GN_EXTRA_ARGS: 'target_cpu = "arm"' GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True' RUN_TESTS: false - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-arm64-debug-fyi: @@ -485,9 +554,7 @@ jobs: GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false' GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True' RUN_TESTS: false - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-arm64-testing-fyi: @@ -496,9 +563,7 @@ jobs: GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false' GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True' RUN_TESTS: false - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps electron-gn-linux-arm64-release-fyi: @@ -507,11 +572,54 @@ jobs: GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false' GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True' RUN_TESTS: false - docker: - - image: electronbuilds/electron:0.0.8 - resource_class: 2xlarge + <<: *gn-linux-build-machine <<: *gn-build-steps + electron-gn-osx-release-fyi: + environment: + GN_CONFIG: //electron/build/args/release.gn + RUN_TESTS: true + <<: *gn-mac-build-machine + <<: *gn-mac-build-steps + + electron-gn-osx-testing-fyi: + environment: + GN_CONFIG: //electron/build/args/testing.gn + RUN_TESTS: true + <<: *gn-mac-build-machine + <<: *gn-mac-build-steps + + electron-gn-osx-debug-fyi: + environment: + GN_CONFIG: //electron/build/args/debug.gn + RUN_TESTS: false + <<: *gn-mac-build-machine + <<: *gn-mac-build-steps + + electron-gn-mas-release-fyi: + environment: + GN_CONFIG: //electron/build/args/release.gn + RUN_TESTS: true + GN_EXTRA_ARGS: 'is_mas_build = true' + <<: *gn-mac-build-machine + <<: *gn-mac-build-steps + + electron-gn-mas-testing-fyi: + environment: + GN_CONFIG: //electron/build/args/testing.gn + RUN_TESTS: true + GN_EXTRA_ARGS: 'is_mas_build = true' + <<: *gn-mac-build-machine + <<: *gn-mac-build-steps + + electron-gn-mas-debug-fyi: + environment: + GN_CONFIG: //electron/build/args/debug.gn + RUN_TESTS: false + GN_EXTRA_ARGS: 'is_mas_build = true' + <<: *gn-mac-build-machine + <<: *gn-mac-build-steps + workflows: version: 2 build-arm: @@ -548,6 +656,12 @@ workflows: - electron-gn-linux-arm-testing-fyi - electron-gn-linux-arm64-debug-fyi - electron-gn-linux-arm64-testing-fyi + build-gn-mac: + jobs: + - electron-gn-mas-debug-fyi + - electron-gn-mas-testing-fyi + - electron-gn-osx-debug-fyi + - electron-gn-osx-testing-fyi nightly-release-test: triggers: @@ -580,3 +694,5 @@ workflows: - electron-gn-linux-ia32-release-fyi - electron-gn-linux-arm-release-fyi - electron-gn-linux-arm64-release-fyi + - electron-gn-mas-release-fyi + - electron-gn-osx-release-fyi diff --git a/BUILD.gn b/BUILD.gn index dd319a34392a..d00c4b1e3d35 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -183,12 +183,6 @@ asar("app2asar") { root = "default_app" } -group("electron") { - deps = [ - ":electron_lib", - ] -} - static_library("electron_lib") { configs += [ "//v8:external_startup_data" ] configs += [ "//third_party/electron_node:node_internals" ] @@ -631,6 +625,21 @@ if (is_mac) { "//ui/strings", ] + data = [] + + data += [ "$root_out_dir/resources.pak" ] + data += [ "$root_out_dir/chrome_100_percent.pak" ] + if (enable_hidpi) { + data += [ "$root_out_dir/chrome_200_percent.pak" ] + } + foreach(locale, locales) { + data += [ "$root_out_dir/locales/$locale.pak" ] + } + + if (!is_mac) { + data += [ "$root_out_dir/resources" ] + } + public_deps = [ "//tools/v8_context_snapshot:v8_context_snapshot", ] @@ -753,3 +762,55 @@ group("chromium_browsertests") { "//content/test:content_browsertests", ] } + +template("dist_zip") { + _runtime_deps_target = "${target_name}__deps" + _runtime_deps_file = + "$root_out_dir/gen.runtime/" + + get_label_info(target_name, "dir") + "/" + + get_label_info(target_name, "name") + ".runtime_deps" + + group(_runtime_deps_target) { + forward_variables_from(invoker, [ "deps", "data_deps", "data" ]) + write_runtime_deps = _runtime_deps_file + } + + action(target_name) { + script = "//electron/build/zip.py" + deps = [ ":$_runtime_deps_target" ] + forward_variables_from(invoker, [ "outputs" ]) + args = rebase_path(outputs + [ _runtime_deps_file ], root_build_dir) + [target_cpu, target_os] + } +} + + +copy("electron_license") { + sources = [ "LICENSE" ] + outputs = [ "$root_build_dir/{{source_file_part}}" ] +} +copy("chromium_licenses") { + deps = [ "//components/resources:about_credits" ] + sources = [ "$root_gen_dir/components/resources/about_credits.html" ] + outputs = [ "$root_build_dir/LICENSES.chromium.html" ] +} + +group("licenses") { + data_deps = [ ":electron_license", ":chromium_licenses" ] +} + +action("electron_version") { + script = "build/write_version.py" + outputs = [ "$root_build_dir/version" ] + args = rebase_path(outputs, root_build_dir) + [ electron_version ] +} + + +dist_zip("electron_dist_zip") { + data_deps = [ ":electron_app", ":licenses", ":electron_version" ] + outputs = [ "$root_build_dir/dist.zip" ] +} + + +group("electron") { + deps = [ ":electron_app" ] +} diff --git a/appveyor-gn.yml b/appveyor-gn.yml index 050d6fc2ad0a..f54a4fff084a 100644 --- a/appveyor-gn.yml +++ b/appveyor-gn.yml @@ -21,6 +21,7 @@ build_script: - md src - ps: Move-Item $env:APPVEYOR_BUILD_FOLDER -Destination src\electron - ps: $env:CHROMIUM_BUILDTOOLS_PATH="$pwd\src\buildtools" + - ps: $env:SCCACHE_PATH="$pwd\src\libchromiumcontent\tools\sccache\aad2120\windows\sccache.exe" - >- gclient config --name "src\electron" @@ -29,8 +30,13 @@ build_script: "https://github.com/electron/electron" - gclient sync --with_branch_heads --with_tags - cd src - - gn gen out/Default "--args=import(\"//electron/build/args/%gn_args%.gn\")" + - >- + %SCCACHE_PATH% -s + - gn gen out/Default "--args=import(\"//electron/build/args/%gn_args%.gn\") cc_wrapper=\"%CC_WRAPPER%\" " - ninja -C out/Default electron:electron_app + - ninja -C out/Default electron:electron_dist_zip + - >- + %SCCACHE_PATH% -s test_script: - ninja -C out/Default electron/build/node:headers - ps: $env:npm_config_nodedir="$pwd/out/Default/gen/node_headers" @@ -43,3 +49,5 @@ test_script: artifacts: - path: test-results.xml name: test-results.xml +- path: src/out/Default/dist.zip + name: dist.zip diff --git a/build/args/release.gn b/build/args/release.gn index 0804548965b6..dabe767ecc7d 100644 --- a/build/args/release.gn +++ b/build/args/release.gn @@ -2,6 +2,7 @@ import("all.gn") is_component_build = false is_component_ffmpeg = true is_official_build = true +strip_debug_info = true # TODO(nornagon): linking non-CFI code (nodejs) with CFI code fails at runtime. # Once we can build nodejs with CFI flags matching Electron's, remove this. diff --git a/build/write_version.py b/build/write_version.py new file mode 100644 index 000000000000..1608074ea2ba --- /dev/null +++ b/build/write_version.py @@ -0,0 +1,11 @@ +import sys + + +def main(argv): + out_file, version = argv + with open(out_file, 'w') as f: + f.write(version) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/build/zip.py b/build/zip.py new file mode 100644 index 000000000000..150656e2c2be --- /dev/null +++ b/build/zip.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +import os +import subprocess +import sys +import zipfile + +LINUX_BINARIES_TO_STRIP = [ + 'electron', + 'libffmpeg.so', + 'libnode.so' +] + +def strip_binaries(target_cpu, dep): + for binary in LINUX_BINARIES_TO_STRIP: + if dep.endswith(binary): + strip_binary(dep, target_cpu) + +def strip_binary(binary_path, target_cpu): + if target_cpu == 'arm': + strip = 'arm-linux-gnueabihf-strip' + elif target_cpu == 'arm64': + strip = 'aarch64-linux-gnu-strip' + elif target_cpu == 'mips64el': + strip = 'mips64el-redhat-linux-strip' + else: + strip = 'strip' + execute([strip, binary_path]) + +def execute(argv): + try: + output = subprocess.check_output(argv, stderr=subprocess.STDOUT) + return output + except subprocess.CalledProcessError as e: + print e.output + raise e + +def main(argv): + dist_zip, runtime_deps, target_cpu, target_os = argv + dist_files = [] + with open(runtime_deps) as f: + for dep in f.readlines(): + dep = dep.strip() + dist_files += [dep] + if sys.platform == 'darwin': + mac_zip_results = execute(['zip', '-r', '-y', dist_zip] + dist_files) + else: + with zipfile.ZipFile(dist_zip, 'w', zipfile.ZIP_DEFLATED) as z: + for dep in dist_files: + if target_os == 'linux': + strip_binaries(target_cpu, dep) + if os.path.isdir(dep): + for root, dirs, files in os.walk(dep): + for file in files: + z.write(os.path.join(root, file)) + else: + z.write(dep) + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/vsts-gn.yml b/vsts-gn.yml index e8bf757b2e13..ddc313edf9a1 100644 --- a/vsts-gn.yml +++ b/vsts-gn.yml @@ -33,8 +33,8 @@ phases: cd src export CHROMIUM_BUILDTOOLS_PATH=`pwd`/buildtools export SCCACHE_WRAPPER="`pwd`/libchromiumcontent/tools/sccache/aad2120/mac/sccache" - "$SCCACHE_WRAPPER" --start-server --azure_container "$(SCCACHE_AZURE_BLOB_CONTAINER)" --azure_connection "$(SCCACHE_AZURE_CONNECTION_STRING)" - "$SCCACHE_WRAPPER" -s + export SCCACHE_HELPER="`pwd`/libchromiumcontent/script/sccache" + "$SCCACHE_HELPER" --start-server --azure_container "$(SCCACHE_AZURE_BLOB_CONTAINER)" --azure_connection "$(SCCACHE_AZURE_CONNECTION_STRING)" echo "##vso[task.setvariable variable=SCCACHE_WRAPPER]$SCCACHE_WRAPPER" echo "##vso[task.setvariable variable=CHROMIUM_BUILDTOOLS_PATH]`pwd`/buildtools" echo "GN gen for: $GN_CONFIG" @@ -59,12 +59,26 @@ phases: (cd electron/spec && npm install) ./out/Default/Electron.app/Contents/MacOS/Electron electron/spec --ci --enable-logging name: Test + condition: and(succeeded(), ne(variables['ELECTRON_RELEASE'], '1')) + + - bash: | + cd src + ninja -C out/Default electron:electron_dist_zip + name: Build_dist_zip + condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1')) - task: PublishTestResults@2 displayName: Publish Test Results inputs: testResultsFiles: '**/test-*.xml' - condition: and(always(), eq(variables['MOCHA_FILE'], 'junit/test-results.xml')) + condition: and(always(), eq(variables['MOCHA_FILE'], 'junit/test-results.xml'), ne(variables['ELECTRON_RELEASE'], '1')) + + - task: PublishBuildArtifacts@1 + displayName: Publish Build Artifacts + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/out/Default/dist.zip' + ArtifactName: dist.zip + condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1')) - bash: | export BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/results?buildId=${BUILD_BUILDID}"