chore: initial work for running releases of GN build - Merge pull request #14831 from electron/prepare-for-4-nightlies

.circleci/config.yml

@@ -25,6 +25,9 @@ env-testing-build: &env-testing-build
 env-release-build: &env-release-build
   GN_CONFIG: //electron/build/args/release.gn
   NOTIFY_SLACK: true
+
+env-publish-build: &env-publish-build
+  GN_CONFIG: //electron/build/args/release.gn
   ELECTRON_RELEASE: 1
 
 env-browsertests: &env-browsertests
@@ -44,11 +47,13 @@ env-arm: &env-arm
   GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True'
   GN_EXTRA_ARGS: 'target_cpu = "arm"'
   MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm
+  BUILD_NATIVE_MKSNAPSHOT: 1
 
 env-arm64: &env-arm64
   GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True'
   GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false'
   MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm64
+  BUILD_NATIVE_MKSNAPSHOT: 1
 
 env-mas: &env-mas
   GN_EXTRA_ARGS: 'is_mas_build = true'
@@ -110,9 +115,11 @@ step-setup-env-for-build: &step-setup-env-for-build
       # To find `gn` executable.
       echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
 
-      # https://github.com/mozilla/sccache
+      if [ "$ELECTRON_RELEASE" != "1" ]; then
-      SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache"
+        # https://github.com/mozilla/sccache
-      echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV
+        SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache"
+        echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV
+      fi
 
 step-install-nodejs-on-mac: &step-install-nodejs-on-mac
   run:
@@ -128,7 +135,11 @@ step-electron-gn-gen: &step-electron-gn-gen
     name: Electron GN gen
     command: |
       cd src
-      gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
+      if [ "$ELECTRON_RELEASE" == "1" ]; then
+        gn gen out/Default --args='import("'$GN_CONFIG'")'" $GN_EXTRA_ARGS"
+      else
+        gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
+      fi
 
 step-electron-build: &step-electron-build
   run:
@@ -194,14 +205,18 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen
     name: ffmpeg GN gen
     command: |
       cd src
-      gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
+      if [ "$ELECTRON_RELEASE" == "1" ]; then
+        gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn")'" $GN_EXTRA_ARGS"
+      else
+        gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
+      fi
 
 step-ffmpeg-build: &step-ffmpeg-build
   run:
     name: Non proprietary ffmpeg build
     command: |
       cd src
-      ninja -C out/ffmpeg third_party/ffmpeg
+      ninja -C out/ffmpeg electron:electron_ffmpeg_zip
 
 step-verify-ffmpeg: &step-verify-ffmpeg
   run:
@@ -210,6 +225,11 @@ step-verify-ffmpeg: &step-verify-ffmpeg
       cd src
       python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg
 
+step-ffmpeg-store: &step-ffmpeg-store
+  store_artifacts:
+    path: src/out/ffmpeg/ffmpeg.zip
+    destination: ffmpeg.zip
+
 step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing
   run:
     name: Setup for headless testing
@@ -236,17 +256,47 @@ step-mksnapshot-store: &step-mksnapshot-store
     path: src/out/Default/mksnapshot.zip
     destination: mksnapshot.zip
 
-step-maybe-generate-breakpad_symbols: &step-maybe-generate-breakpad_symbols
+step-generate-breakpad_symbols: &step-generate-breakpad_symbols
   run:
     name: Generate breakpad symbols
     command: |
-      if [ "$ELECTRON_RELEASE" == "1" ]; then
+      cd src
+      # Build needed dump_syms executable
+      ninja -C out/Default third_party/breakpad:dump_syms
+      electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms"
+      electron/script/zip-symbols.py
+
+step-native-mksnapshot-gn-gen: &step-native-mksnapshot-gn-gen
+  run:
+    name: native mksnapshot GN gen (if needed)
+    command: |
+      if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then
         cd src
-        # Build needed dump_syms executable
+        if [ "$ELECTRON_RELEASE" == "1" ]; then
-        ninja -C out/Default third_party/breakpad:dump_syms
+          gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS"
-        electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms"
+        else
+          gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS"
+        fi
+      else
+        echo 'skipping native mksnapshot GN gen for non arm build'
       fi
 
+step-native-mksnapshot-build: &step-native-mksnapshot-build
+  run:
+    name: native mksnapshot (arm/arm64) build
+    command: |
+      if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then
+        cd src
+        ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip
+        cp out/native_mksnapshot/mksnapshot.zip out/native_mksnapshot/native_mksnapshot.zip
+      else
+        echo 'skipping native mksnapshot build for non arm build'
+      fi
+
+step-native-mksnapshot-store: &step-native-mksnapshot-store
+  store_artifacts:
+    path: src/out/native_mksnapshot/native_mksnapshot.zip
+    destination: native_mksnapshot.zip
 
 # Lists of steps.
 steps-checkout: &steps-checkout
@@ -310,7 +360,6 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests
     - *step-electron-build
     - *step-electron-dist-build
     - *step-electron-dist-store
-    - *step-maybe-generate-breakpad_symbols
 
     # mksnapshot
     - *step-mksnapshot-build
@@ -331,27 +380,47 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests
     - *step-maybe-notify-slack-failure
     - *step-maybe-notify-slack-success
 
+steps-electron-build-for-publish: &steps-electron-build-for-publish
+  steps:
+    - *step-checkout-electron
+    - *step-depot-tools-get
+    - *step-depot-tools-add-to-path
+    - *step-gclient-sync
+    - *step-setup-env-for-build
+
+    # Electron app
+    - *step-electron-gn-gen
+    - *step-electron-build
+    - *step-electron-dist-build
+    - *step-electron-dist-store
+    - *step-generate-breakpad_symbols
+
+    # mksnapshot
+    - *step-mksnapshot-build
+    - *step-mksnapshot-store
+
+    # native_mksnapshot
+    - *step-native-mksnapshot-gn-gen
+    - *step-native-mksnapshot-build
+    - *step-native-mksnapshot-store
+
+    # chromedriver
+    - *step-electron-chromedriver-build
+    - *step-electron-chromedriver-store
+
+    # Node.js headers
+    - *step-nodejs-headers-build
+
+
 steps-native-mksnapshot-build: &steps-native-mksnapshot-build
   steps:
     - attach_workspace:
         at: .
     - *step-depot-tools-add-to-path
     - *step-setup-env-for-build
-
+    - *step-native-mksnapshot-gn-gen
-    - run:
+    - *step-native-mksnapshot-build
-        name: native mksnapshot GN gen
+    - *step-native-mksnapshot-store
-        command: |
-          cd src
-          gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS"
-    - run:
-        name: native mksnapshot (arm/arm64) build
-        command: |
-          cd src
-          ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip
-          cp out/native_mksnapshot/mksnapshot.zip out/native_mksnapshot/native_mksnapshot.zip
-    - store_artifacts:
-        path: src/out/native_mksnapshot/native_mksnapshot.zip
-        destination: native_mksnapshot.zip
 
 steps-ffmpeg-build: &steps-ffmpeg-build
   steps:
@@ -362,14 +431,13 @@ steps-ffmpeg-build: &steps-ffmpeg-build
 
     - *step-ffmpeg-gn-gen
     - *step-ffmpeg-build
+    - *step-ffmpeg-store
 
-    - store_artifacts:
-        path: src/out/ffmpeg/libffmpeg.so
-        destination: libffmpeg.so
     - persist_to_workspace:
         root: .
         paths:
           - src/out/ffmpeg/libffmpeg.so
+          - src/out/ffmpeg/ffmpeg.zip
 
 steps-native-tests: &steps-native-tests
   steps:
@@ -476,9 +544,7 @@ steps-build-mac: &steps-build-mac
     # ffmpeg
     - *step-ffmpeg-gn-gen
     - *step-ffmpeg-build
-    - store_artifacts:
+    - *step-ffmpeg-store
-        path: src/out/ffmpeg/libffmpeg.dylib
-        destination: libffmpeg.dylib
 
     # It would be better to verify ffmpeg as a part of a test job,
     # but it requires `gn` to run, and it's complicated
@@ -547,6 +613,12 @@ jobs:
       <<: *env-release-build
     <<: *steps-electron-build-for-tests
 
+  linux-x64-publish:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-publish-build
+    <<: *steps-electron-build-for-publish
+
   linux-ia32-debug:
     <<: *machine-linux-2xlarge
     environment:
@@ -574,6 +646,13 @@ jobs:
       <<: *env-release-build
     <<: *steps-electron-build-for-tests
 
+  linux-ia32-publish:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-ia32
+      <<: *env-publish-build
+    <<: *steps-electron-build-for-publish
+
   linux-arm-debug:
     <<: *machine-linux-2xlarge
     environment:
@@ -601,6 +680,13 @@ jobs:
       <<: *env-release-build
     <<: *steps-electron-build-for-tests
 
+  linux-arm-publish:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-arm
+      <<: *env-publish-build
+    <<: *steps-electron-build-for-publish
+
   linux-arm-mksnapshot:
     <<: *machine-linux-medium
     environment:
@@ -634,6 +720,13 @@ jobs:
       <<: *env-release-build
     <<: *steps-electron-build-for-tests
 
+  linux-arm64-publish:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-arm64
+      <<: *env-publish-build
+    <<: *steps-electron-build-for-publish
+
   linux-arm64-mksnapshot:
     <<: *machine-linux-medium
     environment:

DEPS

@@ -65,6 +65,33 @@ hooks = [
     'pattern': 'src/electron/package.json',
     'name': 'electron_npm_deps'
   },
+  {
+    'action': [
+      'python',
+      '-c',
+      'import os; os.chdir("src"); os.chdir("electron"); os.system("git submodule update --init --recursive");',
+    ],
+    'pattern': 'src/electron',
+    'name': 'electron_submodules'
+  },
+  {
+    'action': [
+      'python',
+      '-c',
+      'import os; os.chdir("src"); os.chdir("electron"); os.chdir("vendor"); os.chdir("boto"); os.system("python setup.py build");',
+    ],
+    'pattern': 'src/electron',
+    'name': 'setup_boto',
+  },
+  {
+    'action': [
+      'python',
+      '-c',
+      'import os; os.chdir("src"); os.chdir("electron"); os.chdir("vendor"); os.chdir("requests"); os.system("python setup.py build");',
+    ],
+    'pattern': 'src/electron',
+    'name': 'setup_requests',
+  }
 ]
 
 recursedeps = [

appveyor.yml

@@ -38,6 +38,7 @@ build_script:
   - ps: >-
       if ($env:GN_CONFIG -eq 'release') {
         python electron\script\dump-symbols.py -d %cd%/out/Default/electron.breakpad.syms
+        python electron\script\zip-symbols.py
         appveyor PushArtifact out/Default/electron.breakpad.syms
       }
 test_script:
@@ -52,3 +53,16 @@ test_script:
       }
   - cd electron
   - if "%GN_CONFIG%"=="testing" ( echo Running test suite & npm run test -- --ci )
+  - cd ..
+deploy_script:
+  - cd electron
+  - ps: >-
+      if (Test-Path Env:\ELECTRON_RELEASE) {
+        if (Test-Path Env:\UPLOAD_TO_S3) {
+          Write-Output "Uploading Electron release distribution to s3"
+          & python script\upload.py --upload_to_s3
+        } else {
+          Write-Output "Uploading Electron release distribution to github releases"
+          & python script\upload.py
+        }
+      }

docs/development/upgrading-chromium.md

@@ -97,7 +97,7 @@ This is an overview of the steps needed to upgrade Chromium in Electron.
 - Add more build flags to disable features in build-time.
 
 When a Debug build of Electron succeeds, run the tests:
-`$ ./script/test.py`
+`$ npm run test`
 Fix the failing tests.
 
 Follow all the steps above to fix Electron code on all supported platforms.

script/ci-release-build.js

@@ -11,11 +11,10 @@ const appVeyorJobs = {
 }
 
 const circleCIJobs = [
-  'electron-linux-arm',
+  'linux-arm-publish',
-  'electron-linux-arm64',
+  'linux-arm64-publish',
-  'electron-linux-ia32',
+  'linux-ia32-publish',
-  //  'electron-linux-mips64el',
+  'linux-x64-publish'
-  'electron-linux-x64'
 ]
 
 const vstsJobs = [
@@ -94,16 +93,12 @@ function buildAppVeyor (targetBranch, options) {
 
 async function callAppVeyor (targetBranch, job, options) {
   console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`)
-  let environmentVariables = {}
+  const environmentVariables = {
-
+    ELECTRON_RELEASE: 1
-  if (options.ghRelease) {
-    environmentVariables.ELECTRON_RELEASE = 1
-  } else {
-    environmentVariables.RUN_RELEASE_BUILD = 'true'
   }
 
-  if (options.automaticRelease) {
+  if (!options.ghRelease) {
-    environmentVariables.AUTO_RELEASE = 'true'
+    environmentVariables.UPLOAD_TO_S3 = 1
   }
 
   const requestOpts = {
@@ -144,16 +139,14 @@ async function buildVSTS (targetBranch, options) {
     assert(vstsJobs.includes(options.job), `Unknown VSTS CI job name: ${options.job}. Valid values are: ${vstsJobs}.`)
   }
   console.log(`Triggering VSTS to run build on branch: ${targetBranch} with release flag.`)
-  let environmentVariables = {}
+  const environmentVariables = {
+    ELECTRON_RELEASE: 1
+  }
 
   if (!options.ghRelease) {
     environmentVariables.UPLOAD_TO_S3 = 1
   }
 
-  if (options.automaticRelease) {
-    environmentVariables.AUTO_RELEASE = 'true'
-  }
-
   let requestOpts = {
     url: `${vstsURL}/definitions?api-version=4.1`,
     auth: {

script/dump-symbols.py

@@ -5,11 +5,11 @@ import os
 import sys
 
 from lib.config import PLATFORM, enable_verbose_mode, is_verbose_mode
-from lib.util import get_electron_branding, execute, rm_rf
+from lib.util import get_electron_branding, execute, rm_rf, get_out_dir
 
 ELECTRON_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 SOURCE_ROOT = os.path.abspath(os.path.dirname(ELECTRON_ROOT))
-RELEASE_PATH = os.path.join('out', 'Release')
+RELEASE_PATH = get_out_dir()
 
 def main():
   args = parse_args()

script/lib/util.py

@@ -21,6 +21,7 @@ import zipfile
 from lib.config import is_verbose_mode, PLATFORM
 from lib.env_util import get_vs_env
 
+GN_SRC_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', '..'))
 BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor',
                                         'boto'))
 
@@ -295,3 +296,22 @@ def make_version(major, minor, patch, pre = None):
     return major + '.' + minor + '.' + patch
   return major + "." + minor + "." + patch + '-' + pre
 
+def get_out_dir():
+  out_dir = 'Debug'
+  override = os.environ.get('ELECTRON_OUT_DIR')
+  if override is not None:
+    out_dir = override
+  return os.path.join(GN_SRC_DIR, 'out', out_dir)
+
+# NOTE: This path is not created by gn, it is used as a scratch zone by our
+#       upload scripts
+def get_dist_dir():
+  return os.path.join(get_out_dir(), 'gen', 'electron_dist')
+
+def get_electron_exec():
+  if sys.platform == 'darwin':
+    return 'out/{0}/Electron.app/Contents/MacOS/Electron'.format(get_out_dir())
+  elif sys.platform == 'win32':
+    return 'out/{0}/electron.exe'.format(get_out_dir())
+  elif sys.platform == 'linux':
+    return 'out/{0}/electron'.format(get_out_dir())

script/rebuild-test-modules.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import os
-import shutil
-import subprocess
-import sys
-
-from lib.config import PLATFORM, enable_verbose_mode, get_target_arch
-from lib.util import execute_stdout, get_electron_version, safe_mkdir, \
-                     update_node_modules, update_electron_modules
-
-
-SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-
-
-def main():
-  os.chdir(SOURCE_ROOT)
-
-  args = parse_args()
-  config = args.configuration
-
-  if args.verbose:
-    enable_verbose_mode()
-
-  spec_modules = os.path.join(SOURCE_ROOT, 'spec', 'node_modules')
-  out_dir = os.path.join(SOURCE_ROOT, 'out', config)
-  version = get_electron_version()
-  node_dir = os.path.join(out_dir, 'node-{0}'.format(version))
-
-  # Create node headers
-  script_path = os.path.join(SOURCE_ROOT, 'script', 'create-node-headers.py')
-  execute_stdout([sys.executable, script_path, '--version', version,
-                  '--directory', out_dir])
-
-  if PLATFORM == 'win32':
-    lib_dir = os.path.join(node_dir, 'Release')
-    safe_mkdir(lib_dir)
-    iojs_lib = os.path.join(lib_dir, 'iojs.lib')
-    atom_lib = os.path.join(out_dir, 'node.dll.lib')
-    shutil.copy2(atom_lib, iojs_lib)
-    node_lib = os.path.join(lib_dir, 'node.lib')
-    shutil.copy2(atom_lib, node_lib)
-
-  # Native modules can only be compiled against release builds on Windows
-  if config[0] == 'R' or PLATFORM != 'win32':
-    update_electron_modules(os.path.dirname(spec_modules), get_target_arch(),
-                            node_dir)
-  else:
-    update_node_modules(os.path.dirname(spec_modules))
-
-
-def parse_args():
-  parser = argparse.ArgumentParser(description='Rebuild native test modules')
-  parser.add_argument('-v', '--verbose',
-                      action='store_true',
-                      help='Prints the output of the subprocesses')
-  parser.add_argument('-c', '--configuration',
-                      help='Build configuration to rebuild modules against',
-                      default='D',
-                      required=False)
-  return parser.parse_args()
-
-
-if __name__ == '__main__':
-  sys.exit(main())

script/serve-node-headers.py

@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import atexit
-import os
-import shutil
-import sys
-import tarfile
-import time
-
-from subprocess import Popen, PIPE
-from lib.util import execute_stdout
-
-SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-DIST_DIR    = os.path.join(SOURCE_ROOT, 'dist')
-
-
-def main():
-  args = parse_args()
-  header_dir = os.path.join(DIST_DIR, args.version)
-
-  # Generate Headers
-  script_path = os.path.join(SOURCE_ROOT, 'script', 'create-node-headers.py')
-  execute_stdout([sys.executable, script_path, '--version', args.version,
-                  '--directory', header_dir])
-
-  # Launch server
-  script_path = os.path.join(SOURCE_ROOT, 'node_modules', 'serve', 'bin',
-                            'serve.js')
-  server = Popen(['node', script_path, '--port=' + args.port], stdout=PIPE,
-                 cwd=DIST_DIR)
-  def cleanup():
-    server.kill()
-  atexit.register(cleanup)
-
-  time.sleep(1)
-
-  # Generate Checksums
-  script_path = os.path.join(SOURCE_ROOT, 'script', 'upload-node-checksums.py')
-  execute_stdout([sys.executable, script_path, '--version', args.version,
-                  '--dist-url', 'http://localhost:' + args.port,
-                  '--target-dir', header_dir])
-
-  print("Point your npm config at 'http://localhost:" + args.port + "'")
-  server.wait()
-
-def parse_args():
-  parser = argparse.ArgumentParser(description='create node header tarballs')
-  parser.add_argument('-v', '--version', help='Specify the version',
-                      required=True)
-  parser.add_argument('-p', '--port', help='Specify port to run local server',
-                      default='4321')
-  return parser.parse_args()
-
-if __name__ == '__main__':
-  sys.exit(main())

script/upload-index-json.py

@@ -5,10 +5,10 @@ import sys
 import urllib2
 
 from lib.config import s3_config
-from lib.util import s3put, scoped_cwd, safe_mkdir
+from lib.util import s3put, scoped_cwd, safe_mkdir, get_out_dir
 
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-OUT_DIR     = os.path.join(SOURCE_ROOT, 'out', 'D')
+OUT_DIR     = get_out_dir()
 
 BASE_URL = 'https://electron-metadumper.herokuapp.com/?version='
 

script/upload-node-checksums.py

@@ -10,7 +10,6 @@ import tempfile
 from lib.config import s3_config
 from lib.util import download, rm_rf, s3put, safe_mkdir
 
-
 DIST_URL = 'https://atom.io/download/electron/'
 
 

script/upload-node-headers.py

@@ -7,12 +7,19 @@ import shutil
 import sys
 
 from lib.config import PLATFORM, get_target_arch, s3_config
-from lib.util import safe_mkdir, scoped_cwd, s3put
+from lib.util import safe_mkdir, scoped_cwd, s3put, get_out_dir, get_dist_dir
-
 
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-DIST_DIR    = os.path.join(SOURCE_ROOT, 'dist')
+DIST_DIR    = get_dist_dir()
-OUT_DIR     = os.path.join(SOURCE_ROOT, 'out', 'R')
+OUT_DIR     = get_out_dir()
+GEN_DIR     = os.path.join(OUT_DIR, 'gen')
+
+HEADER_TAR_NAMES = [
+  'node-{0}.tar.gz',
+  'node-{0}-headers.tar.gz',
+  'iojs-{0}.tar.gz',
+  'iojs-{0}-headers.tar.gz'
+]
 
 def main():
   args = parse_args()
@@ -30,10 +37,15 @@ def parse_args():
 
 
 def upload_node(bucket, access_key, secret_key, version):
-  with scoped_cwd(DIST_DIR):
+  with scoped_cwd(GEN_DIR):
-    s3put(bucket, access_key, secret_key, DIST_DIR,
+    generated_tar = os.path.join(GEN_DIR, 'node_headers.tar.gz')
+    for header_tar in HEADER_TAR_NAMES:
+      versioned_header_tar = header_tar.format(version)
+      shutil.copy2(generated_tar, os.path.join(GEN_DIR, versioned_header_tar))
+
+    s3put(bucket, access_key, secret_key, GEN_DIR,
           'atom-shell/dist/{0}'.format(version), glob.glob('node-*.tar.gz'))
-    s3put(bucket, access_key, secret_key, DIST_DIR,
+    s3put(bucket, access_key, secret_key, GEN_DIR,
           'atom-shell/dist/{0}'.format(version), glob.glob('iojs-*.tar.gz'))
 
   if PLATFORM == 'win32':
@@ -48,11 +60,11 @@ def upload_node(bucket, access_key, secret_key, version):
     safe_mkdir(os.path.dirname(node_lib))
     safe_mkdir(os.path.dirname(iojs_lib))
 
-    # Copy atom.lib to node.lib, iojs.lib and v4 node.lib
+    # Copy electron.lib to node.lib and iojs.lib.
-    atom_lib = os.path.join(OUT_DIR, 'node.dll.lib')
+    electron_lib = os.path.join(OUT_DIR, 'electron.lib')
-    shutil.copy2(atom_lib, node_lib)
+    shutil.copy2(electron_lib, node_lib)
-    shutil.copy2(atom_lib, iojs_lib)
+    shutil.copy2(electron_lib, iojs_lib)
-    shutil.copy2(atom_lib, v4_node_lib)
+    shutil.copy2(electron_lib, v4_node_lib)
 
     # Upload the node.lib.
     s3put(bucket, access_key, secret_key, DIST_DIR,

script/upload-symbols.py

@@ -5,25 +5,24 @@ import glob
 import sys
 
 from lib.config import PLATFORM, s3_config, enable_verbose_mode
-from lib.util import get_electron_branding, execute, rm_rf, safe_mkdir, s3put
+from lib.util import get_electron_branding, execute, rm_rf, safe_mkdir, s3put, \
-
+                     get_out_dir
 
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-DIST_DIR = os.path.join(SOURCE_ROOT, 'dist')
+RELEASE_DIR = get_out_dir()
-RELEASE_DIR = os.path.join(SOURCE_ROOT, 'out', 'R')
+GEN_DIR = os.path.join(RELEASE_DIR, 'gen')
 
 
 PROJECT_NAME = get_electron_branding()['project_name']
 PRODUCT_NAME = get_electron_branding()['product_name']
 
 if PLATFORM == 'win32':
-  SYMBOLS_DIR = os.path.join(DIST_DIR, 'symbols')
+  SYMBOLS_DIR = os.path.join(GEN_DIR, 'symbols')
 else:
-  SYMBOLS_DIR = os.path.join(DIST_DIR, '{0}.breakpad.syms'.format(PROJECT_NAME))
+  SYMBOLS_DIR = os.path.join(GEN_DIR, '{0}.breakpad.syms'.format(PROJECT_NAME))
 
 PDB_LIST = [
-  os.path.join(RELEASE_DIR, '{0}.exe.pdb'.format(PROJECT_NAME)),
+  os.path.join(RELEASE_DIR, '{0}.exe.pdb'.format(PROJECT_NAME))
-  os.path.join(RELEASE_DIR, 'node.dll.pdb')
 ]
 
 

script/upload.py

@@ -6,6 +6,7 @@ import errno
 import hashlib
 import json
 import os
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -14,7 +15,8 @@ from io import StringIO
 from lib.config import PLATFORM, get_target_arch,  get_env_var, s3_config, \
                        get_zip_name
 from lib.util import get_electron_branding, execute, get_electron_version, \
-                     parse_version, scoped_cwd, s3put
+                     parse_version, scoped_cwd, s3put, get_electron_exec, \
+                     get_out_dir, GN_SRC_DIR
 
 
 ELECTRON_REPO = 'electron/electron'
@@ -24,8 +26,7 @@ PROJECT_NAME = get_electron_branding()['project_name']
 PRODUCT_NAME = get_electron_branding()['product_name']
 
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R')
+OUT_DIR = get_out_dir()
-DIST_DIR = os.path.join(SOURCE_ROOT, 'dist')
 
 DIST_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION)
 SYMBOLS_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'symbols')
@@ -58,40 +59,55 @@ def main():
           'You have to pass --overwrite to overwrite a published release'
 
   # Upload Electron files.
-  upload_electron(release, os.path.join(DIST_DIR, DIST_NAME), args)
+  # Rename dist.zip to  get_zip_name('electron', version, suffix='')
+  electron_zip = os.path.join(OUT_DIR, DIST_NAME)
+  shutil.copy2(os.path.join(OUT_DIR, 'dist.zip'), electron_zip)
+  upload_electron(release, electron_zip, args)
   if get_target_arch() != 'mips64el':
-    upload_electron(release, os.path.join(DIST_DIR, SYMBOLS_NAME), args)
+    symbols_zip = os.path.join(OUT_DIR, SYMBOLS_NAME)
+    shutil.copy2(os.path.join(OUT_DIR, 'symobls.zip'), symbols_zip)
+    upload_electron(release, symbols_zip, args)
   if PLATFORM == 'darwin':
-    api_path = os.path.join(SOURCE_ROOT, 'electron-api.json')
+    api_path = os.path.join(OUT_DIR, 'electron-api.json')
     upload_electron(release, api_path, args)
 
-    ts_defs_path = os.path.join(SOURCE_ROOT, 'electron.d.ts')
+    ts_defs_path = os.path.join(OUT_DIR, 'electron.d.ts')
     upload_electron(release, ts_defs_path, args)
-    
+    dsym_zip = os.path.join(OUT_DIR, DSYM_NAME)
-    upload_electron(release, os.path.join(DIST_DIR, DSYM_NAME), args)
+    shutil.copy2(os.path.join(OUT_DIR, 'dsym.zip'), dsym_zip)
+    upload_electron(release, dsym_zip, args)
   elif PLATFORM == 'win32':
-    upload_electron(release, os.path.join(DIST_DIR, PDB_NAME), args)
+    upload_electron(release, os.path.join(OUT_DIR, PDB_NAME), args)
 
   # Upload free version of ffmpeg.
   ffmpeg = get_zip_name('ffmpeg', ELECTRON_VERSION)
-  upload_electron(release, os.path.join(DIST_DIR, ffmpeg), args)
+  ffmpeg_zip = os.path.join(OUT_DIR, ffmpeg)
+  shutil.copy2(os.path.join(OUT_DIR, 'ffmpeg.zip'), ffmpeg_zip)
+  upload_electron(release, ffmpeg_zip, args)
 
   chromedriver = get_zip_name('chromedriver', ELECTRON_VERSION)
-  upload_electron(release, os.path.join(DIST_DIR, chromedriver), args)
+  chromedriver_zip = os.path.join(OUT_DIR, chromedriver)
-  mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION)
+  shutil.copy2(os.path.join(OUT_DIR, 'chromedriver.zip'), chromedriver_zip)
-  upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args)
+  upload_electron(release, chromedriver_zip, args)
 
+  mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION)
+  mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot)
   if get_target_arch().startswith('arm'):
+    # Upload the native mksnapshot as mksnapshot.zip
+    shutil.copy2(os.path.join(GN_SRC_DIR, 'out', 'native_mksnapshot',
+                              'native_mksnapshot.zip'), mksnapshot_zip)
+    upload_electron(release, mksnapshot_zip, args)
     # Upload the x64 binary for arm/arm64 mksnapshot
     mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION, 'x64')
-    upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args)
+    mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot)
+
+  shutil.copy2(os.path.join(OUT_DIR, 'mksnapshot.zip'), mksnapshot_zip)
+  upload_electron(release, mksnapshot_zip, args)
 
   if not tag_exists and not args.upload_to_s3:
     # Upload symbols to symbol server.
     run_python_script('upload-symbols.py')
     if PLATFORM == 'win32':
-      # Upload node headers.
-      run_python_script('create-node-headers.py', '-v', args.version)
       run_python_script('upload-node-headers.py', '-v', args.version)
 
 
@@ -123,16 +139,7 @@ def get_electron_build_version():
   if get_target_arch().startswith('arm') or os.environ.has_key('CI'):
     # In CI we just build as told.
     return ELECTRON_VERSION
-  if PLATFORM == 'darwin':
+  electron = get_electron_exec()
-    electron = os.path.join(SOURCE_ROOT, 'out', 'R',
-                              '{0}.app'.format(PRODUCT_NAME), 'Contents',
-                              'MacOS', PRODUCT_NAME)
-  elif PLATFORM == 'win32':
-    electron = os.path.join(SOURCE_ROOT, 'out', 'R',
-                              '{0}.exe'.format(PROJECT_NAME))
-  else:
-    electron = os.path.join(SOURCE_ROOT, 'out', 'R', PROJECT_NAME)
-
   return subprocess.check_output([electron, '--version']).strip()
 
 

script/zip-symbols.py

@@ -0,0 +1,37 @@
+import glob
+import os
+import sys
+
+from lib.config import PLATFORM, get_target_arch
+from lib.util import scoped_cwd, get_electron_version, make_zip, \
+                     electron_gyp, get_out_dir
+
+ELECTRON_VERSION = get_electron_version()
+PROJECT_NAME = electron_gyp()['project_name%']
+OUT_DIR = get_out_dir()
+
+def main():
+  if get_target_arch() == 'mips64el':
+    return
+
+  dist_name = 'symbols.zip'
+  zip_file = os.path.join(OUT_DIR, dist_name)
+  licenses = ['LICENSE', 'LICENSES.chromium.html', 'version']
+
+  with scoped_cwd(OUT_DIR):
+    dirs = ['{0}.breakpad.syms'.format(PROJECT_NAME)]
+    make_zip(zip_file, licenses, dirs)
+
+  if PLATFORM == 'darwin':
+    dsym_name = 'dsym.zip'
+    with scoped_cwd(OUT_DIR):
+      dsyms = glob.glob('*.dSYM')
+      make_zip(os.path.join(OUT_DIR, dsym_name), licenses, dsyms)
+  elif PLATFORM == 'win32':
+    pdb_name = 'pdb.zip'
+    with scoped_cwd(OUT_DIR):
+      pdbs = glob.glob('*.pdb')
+      make_zip(os.path.join(OUT_DIR, pdb_name), pdbs + licenses, [])
+
+if __name__ == '__main__':
+  sys.exit(main())

spec/api-notification-dbus-spec.js

@@ -2,7 +2,7 @@
 // with the session bus. This requires python-dbusmock to be installed and
 // running at $DBUS_SESSION_BUS_ADDRESS.
 //
-// script/test.py spawns dbusmock, which sets DBUS_SESSION_BUS_ADDRESS.
+// script/spec-runner.js spawns dbusmock, which sets DBUS_SESSION_BUS_ADDRESS.
 //
 // See https://pypi.python.org/pypi/python-dbusmock to read about dbusmock.
 

spec/api-power-monitor-spec.js

@@ -1,7 +1,7 @@
 // For these tests we use a fake DBus daemon to verify powerMonitor module
 // interaction with the system bus. This requires python-dbusmock installed and
 // running (with the DBUS_SYSTEM_BUS_ADDRESS environment variable set).
-// script/test.py will take care of spawning the fake DBus daemon and setting
+// script/spec-runner.js will take care of spawning the fake DBus daemon and setting
 // DBUS_SYSTEM_BUS_ADDRESS when python-dbusmock is installed.
 //
 // See https://pypi.python.org/pypi/python-dbusmock for more information about

vsts.yml

@@ -61,7 +61,7 @@ jobs:
   - bash: |
       cd src
       gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
-      ninja -C out/ffmpeg third_party/ffmpeg
+      ninja -C out/ffmpeg electron:electron_ffmpeg_zip
     displayName: Non proprietary ffmpeg build
     condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
 
@@ -70,6 +70,7 @@ jobs:
       # Build needed dump_syms executable
       ninja -C out/Default third_party/breakpad:dump_syms
       electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms"
+      electron/script/zip-symbols.py
     displayName: Ninja build app
     condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
 
@@ -115,6 +116,18 @@ jobs:
       ninja -C out/Default electron:electron_mksnapshot_zip
     displayName: Build mksnapshot and zip
 
+  - bash: |
+      cd src/electron
+      if [ "$UPLOAD_TO_S3" != "1" ]; then
+        echo 'Uploading Electron release distribution to github releases'
+        ELECTRON_S3_BUCKET="$(s3_bucket)" ELECTRON_S3_ACCESS_KEY="$(s3_access_key)" ELECTRON_S3_SECRET_KEY="$(s3_secret_key)" ELECTRON_GITHUB_TOKEN="$(github_token)" script/upload.py
+      else
+        echo 'Uploading Electron release distribution to s3'
+        ELECTRON_S3_BUCKET="$(s3_bucket)" ELECTRON_S3_ACCESS_KEY="$(s3_access_key)" ELECTRON_S3_SECRET_KEY="$(s3_secret_key)" ELECTRON_GITHUB_TOKEN="$(github_token)" script/upload.py --upload_to_s3
+      fi
+    name: Upload_distribution
+    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
+
   - task: PublishTestResults@2
     displayName: Publish Test Results
     inputs: