build: [gn] add distributable zip target (#14093)

* build: [gn] add distributable zip target

* build: update the CircleCI config

- enable debug and testing builds on Mac
- run Mac release builds nightly
- run test for Mac release builds
- use shared build machines configs

* Add resources dir to zip on non mac platforms.

.circleci/config.yml

@@ -192,6 +192,11 @@ gn-build-steps: &gn-build-steps
         command: |
           cd src
           ninja -C out/Default electron:electron_app
+    - run:
+        name: Build dist.zip
+        command: |
+          cd src
+          ninja -C out/Default electron:electron_dist_zip
     - run:
         name: Check sccache stats after build
         command: $SCCACHE_WRAPPER -s
@@ -217,6 +222,88 @@ gn-build-steps: &gn-build-steps
         path: src/junit
     - store_artifacts:
         path: src/junit
+    - store_artifacts:
+        path: src/out/Default/dist.zip
+
+gn-mac-build-steps: &gn-mac-build-steps
+  steps:
+    - run:
+        name: Setup depot tools
+        command: |
+          git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
+          echo 'export PATH="$PATH:'"$PWD"'/depot_tools"' >> $BASH_ENV
+          echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
+    - run:
+        name: Install Node.js 10 on MacOS
+        command: |
+          echo 'Installing Node.js 10 for MacOS'
+          brew update
+          brew install node@10
+          brew install gnu-tar
+          echo 'export PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"' >> $BASH_ENV
+    - checkout:
+        path: src/electron
+    - run:
+        name: Gclient sync
+        command: |
+          gclient config \
+            --name "src/electron" \
+            --unmanaged \
+            $GCLIENT_EXTRA_ARGS \
+            "$CIRCLE_REPOSITORY_URL"
+          gclient sync --with_branch_heads --with_tags
+    - run:
+        name: GN gen
+        command: |
+          cd src
+          SCCACHE_PATH="$PWD/libchromiumcontent/tools/sccache/aad2120/mac/sccache"
+          echo 'export SCCACHE_WRAPPER="'"$SCCACHE_PATH"'"' >> $BASH_ENV
+          echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/buildtools"' >> $BASH_ENV
+          source $BASH_ENV
+          gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
+    - run:
+        name: Ninja build
+        command: |
+          cd src
+          ninja -C out/Default electron:electron_app
+    - run:
+        name: Build dist.zip
+        command: |
+          cd src
+          ninja -C out/Default electron:electron_dist_zip
+    - run:
+        name: Check sccache stats after build
+        command: $SCCACHE_WRAPPER -s
+    - run:
+        name: Test
+        environment:
+          MOCHA_REPORTER: mocha-junit-reporter
+          MOCHA_FILE: junit/test-results.xml
+          ELECTRON_DISABLE_SECURITY_WARNINGS: 1
+        command: |
+          if [ "$RUN_TESTS" != "false" ]; then
+            cd src
+            ninja -C out/Default third_party/electron_node:headers
+            export npm_config_nodedir="$PWD/out/Default/gen/node_headers"
+            (cd electron/spec && npm install)
+            ./out/Default/Electron.app/Contents/MacOS/Electron electron/spec --ci --enable-logging
+          fi
+    - store_test_results:
+        path: src/junit
+    - store_artifacts:
+        path: src/junit
+    - store_artifacts:
+        path: src/out/Default/dist.zip
+
+gn-linux-build-machine: &gn-linux-build-machine
+  docker:
+    - image: electronbuilds/electron:0.0.8
+  resource_class: 2xlarge
+
+gn-mac-build-machine: &gn-mac-build-machine
+  macos:
+    xcode: "8.3.3"
+  resource_class: large
 
 build-defaults: &build-defaults
   docker:
@@ -389,27 +476,21 @@ jobs:
       DISPLAY: ':99.0'
       GN_CONFIG: //electron/build/args/debug.gn
       RUN_TESTS: false
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-x64-testing-fyi:
     environment:
       DISPLAY: ':99.0'
       GN_CONFIG: //electron/build/args/testing.gn
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-x64-release-fyi:
     environment:
       DISPLAY: ':99.0'
       GN_CONFIG: //electron/build/args/release.gn
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-ia32-debug-fyi:
@@ -419,9 +500,7 @@ jobs:
       GN_EXTRA_ARGS: 'target_cpu = "x86"'
       NPM_CONFIG_ARCH: ia32
       RUN_TESTS: false
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-ia32-testing-fyi:
@@ -430,9 +509,7 @@ jobs:
       GN_CONFIG: //electron/build/args/testing.gn
       GN_EXTRA_ARGS: 'target_cpu = "x86"'
       NPM_CONFIG_ARCH: ia32
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-ia32-release-fyi:
@@ -441,9 +518,7 @@ jobs:
       GN_CONFIG: //electron/build/args/release.gn
       GN_EXTRA_ARGS: 'target_cpu = "x86"'
       NPM_CONFIG_ARCH: ia32
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-arm-debug-fyi:
@@ -452,9 +527,7 @@ jobs:
       GN_EXTRA_ARGS: 'target_cpu = "arm"'
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True'
       RUN_TESTS: false
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-arm-testing-fyi:
@@ -463,9 +536,7 @@ jobs:
       GN_EXTRA_ARGS: 'target_cpu = "arm"'
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True'
       RUN_TESTS: false
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-arm-release-fyi:
@@ -474,9 +545,7 @@ jobs:
       GN_EXTRA_ARGS: 'target_cpu = "arm"'
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True'
       RUN_TESTS: false
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-arm64-debug-fyi:
@@ -485,9 +554,7 @@ jobs:
       GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false'
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True'
       RUN_TESTS: false
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-arm64-testing-fyi:
@@ -496,9 +563,7 @@ jobs:
       GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false'
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True'
       RUN_TESTS: false
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
   electron-gn-linux-arm64-release-fyi:
@@ -507,11 +572,54 @@ jobs:
       GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false'
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True'
       RUN_TESTS: false
-    docker:
+    <<: *gn-linux-build-machine
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
     <<: *gn-build-steps
 
+  electron-gn-osx-release-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/release.gn
+      RUN_TESTS: true
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-osx-testing-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/testing.gn
+      RUN_TESTS: true
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-osx-debug-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/debug.gn
+      RUN_TESTS: false
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-mas-release-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/release.gn
+      RUN_TESTS: true
+      GN_EXTRA_ARGS: 'is_mas_build = true'
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-mas-testing-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/testing.gn
+      RUN_TESTS: true
+      GN_EXTRA_ARGS: 'is_mas_build = true'
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-mas-debug-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/debug.gn
+      RUN_TESTS: false
+      GN_EXTRA_ARGS: 'is_mas_build = true'
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
 workflows:
   version: 2
   build-arm:
@@ -548,6 +656,12 @@ workflows:
       - electron-gn-linux-arm-testing-fyi
       - electron-gn-linux-arm64-debug-fyi
       - electron-gn-linux-arm64-testing-fyi
+  build-gn-mac:
+    jobs:
+      - electron-gn-mas-debug-fyi
+      - electron-gn-mas-testing-fyi
+      - electron-gn-osx-debug-fyi
+      - electron-gn-osx-testing-fyi
 
   nightly-release-test:
     triggers:
@@ -580,3 +694,5 @@ workflows:
       - electron-gn-linux-ia32-release-fyi
       - electron-gn-linux-arm-release-fyi
       - electron-gn-linux-arm64-release-fyi
+      - electron-gn-mas-release-fyi
+      - electron-gn-osx-release-fyi

BUILD.gn

@@ -183,12 +183,6 @@ asar("app2asar") {
   root = "default_app"
 }
 
-group("electron") {
-  deps = [
-    ":electron_lib",
-  ]
-}
-
 static_library("electron_lib") {
   configs += [ "//v8:external_startup_data" ]
   configs += [ "//third_party/electron_node:node_internals" ]
@@ -631,6 +625,21 @@ if (is_mac) {
       "//ui/strings",
     ]
 
+    data = []
+
+    data += [ "$root_out_dir/resources.pak" ]
+    data += [ "$root_out_dir/chrome_100_percent.pak" ]
+    if (enable_hidpi) {
+      data += [ "$root_out_dir/chrome_200_percent.pak" ]
+    }
+    foreach(locale, locales) {
+      data += [ "$root_out_dir/locales/$locale.pak" ]
+    }
+
+    if (!is_mac) {
+      data += [ "$root_out_dir/resources" ]
+    }
+
     public_deps = [
       "//tools/v8_context_snapshot:v8_context_snapshot",
     ]
@@ -753,3 +762,55 @@ group("chromium_browsertests") {
     "//content/test:content_browsertests",
   ]
 }
+
+template("dist_zip") {
+  _runtime_deps_target = "${target_name}__deps"
+  _runtime_deps_file =
+      "$root_out_dir/gen.runtime/" +
+      get_label_info(target_name, "dir") + "/" +
+      get_label_info(target_name, "name") + ".runtime_deps"
+
+  group(_runtime_deps_target) {
+    forward_variables_from(invoker, [ "deps", "data_deps", "data" ])
+    write_runtime_deps = _runtime_deps_file
+  }
+
+  action(target_name) {
+    script = "//electron/build/zip.py"
+    deps = [ ":$_runtime_deps_target" ]
+    forward_variables_from(invoker, [ "outputs" ])
+    args =  rebase_path(outputs + [ _runtime_deps_file ], root_build_dir) + [target_cpu, target_os]
+  }
+}
+
+
+copy("electron_license") {
+  sources = [ "LICENSE" ]
+  outputs = [ "$root_build_dir/{{source_file_part}}" ]
+}
+copy("chromium_licenses") {
+  deps = [ "//components/resources:about_credits" ]
+  sources = [ "$root_gen_dir/components/resources/about_credits.html" ]
+  outputs = [ "$root_build_dir/LICENSES.chromium.html" ]
+}
+
+group("licenses") {
+  data_deps = [ ":electron_license", ":chromium_licenses" ]
+}
+
+action("electron_version") {
+  script = "build/write_version.py"
+  outputs = [ "$root_build_dir/version" ]
+  args = rebase_path(outputs, root_build_dir) + [ electron_version ]
+}
+
+
+dist_zip("electron_dist_zip") {
+  data_deps = [ ":electron_app", ":licenses", ":electron_version" ]
+  outputs = [ "$root_build_dir/dist.zip" ]
+}
+
+
+group("electron") {
+  deps = [ ":electron_app" ]
+}

appveyor-gn.yml

@@ -21,6 +21,7 @@ build_script:
   - md src
   - ps: Move-Item $env:APPVEYOR_BUILD_FOLDER -Destination src\electron
   - ps: $env:CHROMIUM_BUILDTOOLS_PATH="$pwd\src\buildtools"
+  - ps: $env:SCCACHE_PATH="$pwd\src\libchromiumcontent\tools\sccache\aad2120\windows\sccache.exe"
   - >-
       gclient config
       --name "src\electron"
@@ -29,8 +30,13 @@ build_script:
       "https://github.com/electron/electron"
   - gclient sync --with_branch_heads --with_tags
   - cd src
-  - gn gen out/Default "--args=import(\"//electron/build/args/%gn_args%.gn\")"
+  - >-
+      %SCCACHE_PATH% -s
+  - gn gen out/Default "--args=import(\"//electron/build/args/%gn_args%.gn\") cc_wrapper=\"%CC_WRAPPER%\" "
   - ninja -C out/Default electron:electron_app
+  - ninja -C out/Default electron:electron_dist_zip
+  - >-
+      %SCCACHE_PATH% -s
 test_script:
   - ninja -C out/Default electron/build/node:headers
   - ps: $env:npm_config_nodedir="$pwd/out/Default/gen/node_headers"
@@ -43,3 +49,5 @@ test_script:
 artifacts:
 - path: test-results.xml
   name: test-results.xml
+- path: src/out/Default/dist.zip
+  name: dist.zip

build/args/release.gn

@@ -2,6 +2,7 @@ import("all.gn")
 is_component_build = false
 is_component_ffmpeg = true
 is_official_build = true
+strip_debug_info = true
 
 # TODO(nornagon): linking non-CFI code (nodejs) with CFI code fails at runtime.
 # Once we can build nodejs with CFI flags matching Electron's, remove this.

build/write_version.py

@@ -0,0 +1,11 @@
+import sys
+
+
+def main(argv):
+  out_file, version = argv
+  with open(out_file, 'w') as f:
+    f.write(version)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

build/zip.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+import os
+import subprocess
+import sys
+import zipfile
+
+LINUX_BINARIES_TO_STRIP = [
+  'electron',
+  'libffmpeg.so',
+  'libnode.so'
+]
+
+def strip_binaries(target_cpu, dep):
+  for binary in LINUX_BINARIES_TO_STRIP:
+    if dep.endswith(binary):
+     strip_binary(dep, target_cpu)
+
+def strip_binary(binary_path, target_cpu):
+  if target_cpu == 'arm':
+    strip = 'arm-linux-gnueabihf-strip'
+  elif target_cpu == 'arm64':
+    strip = 'aarch64-linux-gnu-strip'
+  elif target_cpu == 'mips64el':
+    strip = 'mips64el-redhat-linux-strip'
+  else:
+    strip = 'strip'
+  execute([strip, binary_path])
+
+def execute(argv):
+  try:
+    output = subprocess.check_output(argv, stderr=subprocess.STDOUT)
+    return output
+  except subprocess.CalledProcessError as e:
+    print e.output
+    raise e
+
+def main(argv):
+  dist_zip, runtime_deps, target_cpu, target_os = argv
+  dist_files = []
+  with open(runtime_deps) as f:
+    for dep in f.readlines():
+      dep = dep.strip()
+      dist_files += [dep]
+  if sys.platform == 'darwin':
+    mac_zip_results = execute(['zip', '-r', '-y', dist_zip] + dist_files)
+  else:
+    with zipfile.ZipFile(dist_zip, 'w', zipfile.ZIP_DEFLATED) as z:
+      for dep in dist_files:
+        if target_os == 'linux':
+            strip_binaries(target_cpu, dep)
+        if os.path.isdir(dep):
+          for root, dirs, files in os.walk(dep):
+            for file in files:
+              z.write(os.path.join(root, file))
+        else:
+          z.write(dep)
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

vsts-gn.yml

@@ -33,8 +33,8 @@ phases:
       cd src
       export CHROMIUM_BUILDTOOLS_PATH=`pwd`/buildtools
       export SCCACHE_WRAPPER="`pwd`/libchromiumcontent/tools/sccache/aad2120/mac/sccache"
-      "$SCCACHE_WRAPPER" --start-server --azure_container "$(SCCACHE_AZURE_BLOB_CONTAINER)" --azure_connection "$(SCCACHE_AZURE_CONNECTION_STRING)"
+      export SCCACHE_HELPER="`pwd`/libchromiumcontent/script/sccache"
-      "$SCCACHE_WRAPPER" -s
+      "$SCCACHE_HELPER" --start-server --azure_container "$(SCCACHE_AZURE_BLOB_CONTAINER)" --azure_connection "$(SCCACHE_AZURE_CONNECTION_STRING)"
       echo "##vso[task.setvariable variable=SCCACHE_WRAPPER]$SCCACHE_WRAPPER"
       echo "##vso[task.setvariable variable=CHROMIUM_BUILDTOOLS_PATH]`pwd`/buildtools"
       echo "GN gen for: $GN_CONFIG"
@@ -59,12 +59,26 @@ phases:
       (cd electron/spec && npm install)
       ./out/Default/Electron.app/Contents/MacOS/Electron electron/spec --ci --enable-logging
     name: Test
+    condition: and(succeeded(), ne(variables['ELECTRON_RELEASE'], '1'))
+
+  - bash: |
+      cd src
+      ninja -C out/Default electron:electron_dist_zip
+    name: Build_dist_zip
+    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
 
   - task: PublishTestResults@2
     displayName: Publish Test Results
     inputs:
       testResultsFiles: '**/test-*.xml'
-    condition: and(always(), eq(variables['MOCHA_FILE'], 'junit/test-results.xml'))
+    condition: and(always(), eq(variables['MOCHA_FILE'], 'junit/test-results.xml'), ne(variables['ELECTRON_RELEASE'], '1'))
+
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Build Artifacts
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/out/Default/dist.zip'
+      ArtifactName: dist.zip
+    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
 
   - bash: |
       export BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/results?buildId=${BUILD_BUILDID}"